{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "plt.rcParams['figure.figsize'] = (4, 4)\n",
    "plt.rcParams['figure.dpi'] = 150\n",
    "plt.rcParams['lines.linewidth'] = 3\n",
    "sns.set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = sns.load_dataset(\"titanic\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "p3_df = df[[\"age\", \"fare\", \"pclass\", \"sex\", \"survived\"]].copy()\n",
    "p3_df = p3_df.dropna()\n",
    "p3_df['sex'] = p3_df['sex'].replace(\"male\", 0)\n",
    "p3_df['sex'] = p3_df['sex'].replace(\"female\", 1)\n",
    "np.random.seed(23)\n",
    "p3_train, p3_validation = np.split(p3_df, [600])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>fare</th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>survived</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>38.0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>26.0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>35.0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>35.0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    age     fare  pclass  sex  survived\n",
       "0  22.0   7.2500       3    0         0\n",
       "1  38.0  71.2833       1    1         1\n",
       "2  26.0   7.9250       3    1         1\n",
       "3  35.0  53.1000       1    1         1\n",
       "4  35.0   8.0500       3    0         0"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p3_df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Regularization vs. No-Regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import LogisticRegressionCV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In Homework 5, we had you build a Logistic Regression model with polynomial degree 2."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "p3c_model = Pipeline([('scale', StandardScaler()),\n",
    "                      ('poly', PolynomialFeatures(degree=2)),\n",
    "                      ('model',\n",
    "                       LogisticRegression(fit_intercept=False,\n",
    "                                          penalty='none',\n",
    "                                          solver='lbfgs'))])\n",
    "p3c_model.fit(p3_train[[\"fare\", \"age\", \"pclass\", \"sex\"]], p3_train[\"survived\"])\n",
    ";"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The coefficients are given below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.13102662, -1.49936142, -0.60810606, -1.63412315,  1.65926409,\n",
       "         0.16231128,  0.42236562, -0.1055555 , -0.25525431,  0.24084911,\n",
       "         0.37054654,  0.22395218,  0.70626524, -0.8443333 , -0.18805324]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p3c_model.named_steps['model'].coef_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can regularize the model by simply removing `penalty = 'none'`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "p3c_model_regularized = Pipeline([('scale', StandardScaler()),\n",
    "                                  ('poly', PolynomialFeatures(degree=2)),\n",
    "                                  ('model',\n",
    "                                   LogisticRegression(fit_intercept=False,\n",
    "                                                      solver='lbfgs'))])\n",
    "p3c_model_regularized.fit(p3_train[[\"fare\", \"age\", \"pclass\", \"sex\"]],\n",
    "                          p3_train[\"survived\"])\n",
    ";"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Comparing our coefficients, we see that they are slightly smaller overall in the regularized version."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.13102662, -1.49936142, -0.60810606, -1.63412315,  1.65926409,\n",
       "         0.16231128,  0.42236562, -0.1055555 , -0.25525431,  0.24084911,\n",
       "         0.37054654,  0.22395218,  0.70626524, -0.8443333 , -0.18805324]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p3c_model.named_steps['model'].coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.93868941, -0.95516024, -0.54467238, -1.41258457,  1.49788079,\n",
       "         0.13732684,  0.38516578,  0.11286982, -0.21226869,  0.19729581,\n",
       "         0.32346106,  0.20084613,  0.5685718 , -0.73936734, -0.08743148]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p3c_model_regularized.named_steps['model'].coef_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Just as with `LinearRegression`, we can tune the strength of the regularization. Instead of `alpha`, the sklearn authors used the parameter `C`.\n",
    "\n",
    "Note that `alpha` was proportional to regularization strength. For some reason, the authors of sklearn decided to make `C` inversely proportional to regularization strength.\n",
    "\n",
    "1. \"Strong\" regularization (i.e. small coefficients): Large `alpha`, small `C`\n",
    "2. \"Weak\" regularization: Small `alpha`, large `C`\n",
    "\n",
    "For example, if we set `C` = 1e10, we'll see that the coefficients are the same as the unregularized version."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "huge_c_model = Pipeline([('scale', StandardScaler()),\n",
    "                         ('poly', PolynomialFeatures(degree=2)),\n",
    "                         ('model',\n",
    "                          LogisticRegression(fit_intercept=False,\n",
    "                                             C=1e10,\n",
    "                                             solver='lbfgs'))])\n",
    "huge_c_model.fit(p3_train[[\"fare\", \"age\", \"pclass\", \"sex\"]],\n",
    "                 p3_train[\"survived\"])\n",
    ";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.13102662, -1.49936142, -0.60810606, -1.63412315,  1.65926409,\n",
       "         0.16231128,  0.42236562, -0.1055555 , -0.25525431,  0.24084911,\n",
       "         0.37054654,  0.22395218,  0.70626524, -0.8443333 , -0.18805324]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p3c_model.named_steps['model'].coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.13102662, -1.49936142, -0.60810606, -1.63412315,  1.65926409,\n",
       "         0.16231128,  0.42236562, -0.1055555 , -0.25525431,  0.24084911,\n",
       "         0.37054654,  0.22395218,  0.70626524, -0.8443333 , -0.18805324]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "huge_c_model.named_steps['model'].coef_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Whereas if we pick a tiny `C`, e.g. 1e-8, the model is strongly regularized and the coefficients are very small."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "tiny_c_model = Pipeline([('scale', StandardScaler()),\n",
    "                         ('poly', PolynomialFeatures(degree=2)),\n",
    "                         ('model',\n",
    "                          LogisticRegression(fit_intercept=False,\n",
    "                                             C=1e-7,\n",
    "                                             solver='lbfgs'))])\n",
    "tiny_c_model.fit(p3_train[[\"fare\", \"age\", \"pclass\", \"sex\"]],\n",
    "                 p3_train[\"survived\"])\n",
    ";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-5.60010955e-06,  7.51111139e-06, -2.65116592e-06,\n",
       "        -1.03234319e-05,  1.56655089e-05,  1.54931392e-05,\n",
       "         1.14096696e-06, -3.34372037e-06,  1.91122677e-06,\n",
       "        -5.34841905e-06,  2.87967040e-06,  1.77226488e-06,\n",
       "        -2.00775812e-06, -3.98826178e-06,  3.30272756e-06]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tiny_c_model.named_steps['model'].coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_titanic_accuracy(model, data):\n",
    "    y_hat_class = model.predict(data[[\"fare\", \"age\", \"pclass\", \"sex\"]])\n",
    "    y_obs = data[\"survived\"]\n",
    "    return sum(y_hat_class == y_obs) / len(y_hat_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7716666666666666"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_titanic_accuracy(tiny_c_model, p3_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8333333333333334"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_titanic_accuracy(huge_c_model, p3_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_train_and_validation_accuracy(C):\n",
    "    c_model = Pipeline([('scale', StandardScaler()),\n",
    "                        ('poly', PolynomialFeatures(degree=2)),\n",
    "                        ('model',\n",
    "                         LogisticRegression(fit_intercept=False,\n",
    "                                            C=C,\n",
    "                                            max_iter=10000,\n",
    "                                            solver='lbfgs'))])\n",
    "    c_model.fit(p3_train[[\"fare\", \"age\", \"pclass\", \"sex\"]],\n",
    "                p3_train[\"survived\"])\n",
    "\n",
    "    training_accuracy = compute_titanic_accuracy(c_model, p3_train)\n",
    "    val_accuracy = compute_titanic_accuracy(c_model, p3_validation)\n",
    "\n",
    "    return training_accuracy, val_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7766666666666666, 0.8070175438596491)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_train_and_validation_accuracy(1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracies = [\n",
    "    get_train_and_validation_accuracy(C) for C in 10**np.linspace(-7, 5, 100)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8508771929824561"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(np.array(accuracies)[:, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEQCAYAAACugzM1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZhjVZn48e+9Se1Lal96p5vu0xvdCM1OA8omiCADMjMMiKMMOjqgAv5EBXRE1FFUdAR3xg1UhBkYBUQYtgZktykaqg9N01R3de1bUnsquff3x03SqT1JJZVK+v08D08nN/fenEOl6s3Z3mPYto0QQggxHTPdBRBCCLGwSaAQQggxIwkUQgghZiSBQgghxIwkUAghhJiRO90FSLI84CigFQimuSxCCJEpXEA98CIwOvHFbAsURwHb0l0IIYTIUFuBpycezLZA0QrQ2zuIZS3c9SGVlcV0dw+kuxhJkS11yZZ6gNRloVrIdTFNg/LyIgj9DZ0o2wJFEMCy7AUdKIAFX754ZEtdsqUeIHVZqDKgLlN22ctgthBCiBlJoBBCCDGjbOt6EkJkCdu26ejooLOzG8vK/EmMHR0mlmWluxi43bmUl1fjcsX+518ChRBiQert7cTtNqmoqMXlcmMYRrqLNCdut0kgkN5AYds2g4M+ens7qaqqj/k66XoSQixIfv8I5eVVuN05GR8kFgrDMCgqKiUQ8Md1nbQohJgDe3QQOziGWVg2/nhglGDbW4Azy8Uo9OCqWJqGEmYyG8MwgfR312STRIKuBAoh5mDkmd9g9TZTdMFN4477tz+I/5X7DxwwXBRf9gOM3IJ5LqEQcyddT0LMgdWzD6unBdsKjD/eux+jpIrCc79I7hHngR3EHvalqZQiGQYGBvj856+N+fydO9/gG9+4afYTM4C0KIRIkG1bWN52Jwj0d2F46iKvWd42zPLFuOpWY/sHnfNHB4DaNJVWzFV/v49du3TM569du57rrlufwhLNHwkUQiTIHuyF4BgAlrcdMxQonADSQc7iDQAYecXO8ZHB9BRUJMWtt36Lrq5OPv/5a2lq2oPHU0ZeXh433/xNvv71m+js7KCrq5MtW47muutu4G9/e5k77vgJP/jBT/i3f7uCDRs2sn373+jr6+XTn/4sxx13QrqrFDMJFEIkyPK2Rz1uAzYD4QDix/Q4rYdIoBhdmHl+MsUzr7XydMOUqYjm7MRN9Zxw2MzTRT/96c9y5ZUf46qrruaDHzyXP/zhP6mvX8Qjj/yZ1avX8NWv/gdjY2NccskH0XrnpOsDgTF+/OP/4umnn+KnP/2hBAohDgZOcAAMc0LQcB6HWxhGfrhFIYEiW5SXV1BfvwiA009/L2+8sYO7776Ld97Zg9frZXh4aNI1xx57PAArV66ivz+zxqskUAiRIMvbDq5czPJF0wSK0HhEbiFgSItijk44bPZv/fMlLy8v8viee37HE088xrnnns+FFx7Nnj27se3Jyf9yc51rDMOY8vWFTGY9CZEgy9uG6anBLKs70LoIHceVg1FUDoBhmpBXKGMUGc7lchEMTk4l8uKLz3PuuX/HGWechd/vZ9euNxdEqo5kkhaFEAmyve2YFUswPXUE3noeO+DHcOeGBrZrQ4vFHEZesbQoMlxFRSW1tXV87Wv/Pu74RRddzC23fJ3f/Oa/KCoqZuPGTbS2trB48ZI0lTT5JFAIkQDbCmL5OnEfcmSoi8nG8nXiqliMHZoaG83IL5Ixigzndrv50Y/umHT8yCOP4re//e8przniiC0A/OAHP4nkeqqvX8Q99/wxpWVNNul6EiIB9kA32EFMT11k0NrytTkBpL8Ts6xu3PnSohCZTAKFEAmw+pwxCcNTGxm0tvranQBiBTFLxy+sM/KKsEdljEJkJgkUQiTA8h2YAmvkFmIUlGL72iKD2sbEFkV+sXQ9iYwlgUKIBFh9bZBTgJFfAoBZWovlbT8wNXZiiyK/GMZGsIOBSfcSYqGTQCFEAixfO2ZZXSRls+GpCwWKNsjJxygoHXf+gdXZ0v0kMo8ECiESYHnbxrUaTE8t9lAfwa4mpztqQs5/I68IkDQeIjNJoBAiTnZwDLu/+8DKaw6swrY6do87HiZpPEQmk0AhRJwsXwdgTwgUocFr2z7wOEokUEiL4qBw881f5sEH/0hXVyfXXnvVlOeceOKWGe/R0rKfr3/9K0D697aQBXdCxGli0j/ncU3U4ylaFKGuJySNx0GlqqqaW275fkLXtrW1sn9/M5D+vS0kUAgRJzs0BTY6IBjuPIyiCuzBnqlbFJJqfM7G3nyGMf1USu6do04iZ83Mab+/8IXPcsYZ7+WUU04F4CMfuYQrr/wMP/nJ7YyOjtDfP8BVV32GrVtPiVzT2trClVd+jHvu+SMtLS186UtfZHh4mA0bNkbO6ezs4Otfv4mBgX66ujo5++z3c/nlH+d737uFlpb9fPvb/8G7331qZG+LvXub+OY3b6a/30d+fgGf/vS1rFu3gZtv/jJFRcVo3UhXVycf/vDlvO995ybl/490PQkRJ8vbjpFfcqCVEBIOHFO1KMjJB9MlYxQZ7Mwzz+bRRx8GYN++vfj9fu699/dcd90N3HHHnVx33fX89Kc/nPb6b3/7G5x99vv5xS/u4rDDNkeOP/LIw5x++pn85Ce/4Fe/+j133/1b+vr6+NSnrkWpdVxzzefG3eemm27ggx/8B375y99x5ZVXc/31n8Pv9wPQ0dHO7bf/jG984zvcdtv3klZ3aVEIEServwujtHrScbNqOZavY1IAASe1tJPGQ7qeEpWz5oRZv/Wn0vHHn8h3v/tNhoYGefTRhznzzLO46KKLefbZbTz++KO8/vprDA8PT3v9K6+8zJe+dDMAZ5xxVmTM4eKLL+WVV17irrt+zZ49uwkExhgZmfo+Q0NDNDc3c/LJ7wFg48bDKC0tZe/eJgCOPvoYDMNg5cpV+HzepNVdWhRCxMkeG8HIKZh0PG/L+RSe/6Vpr5PEgJktJyeHE07YytNPP8Vjjz3C6ae/l09+8l9obHwdpdbyoQ99ZMZ9JgzDwLLsyGPTdAHwn//5Xf7wh99RV1fPZZd9FI+nbNr72Pbk9OW2TST9efSeF8mU0kChlLpYKfWGUmqXUuqTU7x+hFLqRaXUq0qpPymlyia8vkQp1aOUWpHKcgoRl6CTTnwiw52HOWGh3bjXJTFgxjvzzLP53e9+g8dTRmFhIfv2NfHRj36cY489gW3bnpxxH4qjjjqGhx9+EIAnn3wMv38UgJdeep6LL76U97znNPbubaKzswPLsnC53JP2vygqKmbRosU8+eRjAOzY8Ro9Pd2sXLkqRTV2pCxQKKUWAzcDJwKHA1copSYO238PuFFrvRnQwLVR15vAz4DJv5FCpJEdGIMpAsVsnHxP0vWUyTZtOpyBgQHOOOMsSks9nHPOeVx66UX80z9dyNDQECMjI9N2P11zzed48snHuOyyf+Svf32GwkKni/KSSz7MTTfdyKWXXsS9997N2rXraWnZz4oVKxgY6Oemm24Yd58bb7yJP/zhd3zoQ3/Pd7/7TW6++Zvk5OSktN5GqrbkU0pdBpyktf5o6PkNgKG1/krUOc8C39Na/14p9UOgWWt9c+i164AA8G/AKVrrd2J42xXAnu7ugUgTbyGqri6hs7M/3cVIimypSzz1GLjzalyLN1Bwykfjeo+RJ+8gsK+B4ktuTaSIMcuWn0lbWxNLlhxCIJAdu8WF96NYCNramqirWx55bpoGlZXFAIcA70w8P5VdT4uA1qjnrcDELZ+uBn6qlGoFTgd+BKCUOhJ4D/CdFJZPiMQE/BjuBL7B5RVhjw5k3H7JQqRy1pMJRP9GGEAknCqlCoCfA6dprV9QSl0N/Eop9UHgduCDWmtLKRX3G4ci44JWXV2S7iIkTbbUJdZ6DFhjFJYWUxlnvfsqK+kJBqgqy8XMzU+kiDHLhp9JR4fzPdbtzp45NwulLqZpxvUZSWWgaAa2Rj2vA1qinm8EhrXWL4Se/xi4KXRNLfC/oSCxCHhQKXW+1lrH8sbS9TR/sqUusdbDtm3ssVGG/cRdb3/QaYV07m/DLK5MqJyxyJafiWXZ2LbFhPHcjLVQup5s28ayrHGfkaiupymlMlA8CnxZKVUNDAIXAFdEvf4WsFQppUIB4DzgRa31wzhjDQAopd4Bzo5xjEKI1AqOOf8mMpgdziA7MgApDBTZIjc3n56eLoqKPLhc7qRP+TwY2bbN4KAPd5yf35QFCq31fqXUF4HHcWYu/SzUxfQgzkynl5RSHwbuVkoZQAfwz6kqjxBJEQoUhiuRQCF7UsSjvLwaGKGzsx3LyvxmhWmaM06fnS9ud27o/20c16SoLABore8C7ppw7Oyoxw8BD81yjxUpKZwQCbADTqqERKfHgqQaj5VhGFRX12AYkxc3ZqJM7hJcGCMrQmSKUKCYasHdbCTVuMhUEiiEiEOkReGKf3rsuDEKITKIBAoh4hGcQ4vClQPuPBmjEBlHAoUQcZjLGAWE03hIi0JkFgkUQsRjDmMU4HQ/yRiFyDSyH4UQcTgwRjGHFsU8dj0NjwZoapt9po2nOJf6ysn7aAgBEiiEiE94HUXCLYpirIG9ySzRtIZHA9z0y5do6xma9VyXafDNfz2e8pK8eSiZyDQSKISIQzLGKJiHFoVt2/zXQztp7x3io+9bR2Xp9LmlBobHuP2+HTy7o5X3Hbci5WUTmUcChRDxSNIYhW1bGEbqhggfeamZl3Z2cOEpqzjhsPpZz1+zxMPTDa2cfexySZUhJpFAIUQc7GDi6ygglMbDtsE/DFPsrR1medsxSqsTCiZv7OnmD4+/xeGHVnHWMctiumbr5kX8/IFGdjV7WbO0bNLrtm3T1N7P8Egg7vLMRUvfCN6+2bvOMsF81GVZXQlF+cnfxEgChRDxCPgBI/FAUeCkdraHfZEFeBNZ/Z0M3n0d+ad+gpyVR8V1f9u2uf2eVykvyePyc9bF3DrYomq485E32dbQMmWgePiFfdz9+FtxlUXMv6PX1fDx8zYm/b4SKISIgx3wgzsn4e4Zs8RJxmb52jHLpu4SsnqawbaxuvdCnIHi7VYfTW39fOi9isI4vlnm5bo4el0tz73RxsWnraEg78CfBr23l3ue2M27VldxxlFL4yrPXJWVFdKXJS2K+ajL0prU7EMigUKIeAT8CWWODTM8tYDTtTSd8GsznTOdpxtayc1xccy62riv3bq5nqdebeGFxnZOPnwxAH0Do/zo/tepLi/g8nPWjwsg8yGTE+lNlMl1kUAhRBzswFjCM54AjPwSyC2cJVC0jfs3VqP+IM+/0c6Jmxcl9Ad9ZX0pi6qKeOJvLVR5nIytf3z2HYZHA1zz94fPe5AQC4f85IWIR9A/t0BhGJie2phbFLZtx9zN9ZLuYMQf5PSjYxvAnqpsJ29exG//bxff/v32yPHLz1nHkpqFv72wSB0JFELEI+DHcM9tVonpqSPY9ua0r1vedjBMCIxiD3sxCicPLk9lW0MrNeUFbFhZSVdXYmlC3nPkYlYuLo1sJVxckCMrtoXkehIiHnbAn3D6jjDTU4s90HNg8d64+49iD/bgqj0UAKsvtu6n9p4h3tzXx9ZN9XNaB+EyTVYt8rB6SRmrl5RJkBCABAoh4hMcS3ixXZjpqQNsLF/npNcsXwcArmWbQ89jG9B++rVWDAOO3zj74joh4iWBQog42IHROY1RgNOigKkHq8MtCPfi9WC6Y2pRBC2Lp19r5bCVlZKrSaSEBAoh4hEYczYgmgNzhimy4RaE6anD9NRgx9Ci2PF2D94BP1s3LZpTuYSYjgQKIeJgB/3gntu3diO3EKOgFHvKFkU7RoEHI7cAs7Q2pimy2xpaKSnMYfOhlXMqlxDTkUAhRDySMOsJnBbDVOMPtq8ds6wOAMNTh+XrwLasae/jG/Tz6ltdHL+xDrdLfp1FasgnS4g4OCk85j4OYJTWTjn+YHnbMEudrinTUwvBAPZg97T3eXZHG0HL5kTpdhIpJIFCiHgE/HMeowAwy2qxh73Y/uHIMds/5CQL9DgtCjP073SL82zb5unXWlm1qJTFVTKNVaSOBAohYmRbAbCtOc96AiKthujup3BACA92zzQ7CuDtFh8tXYNs3SytCZFasjJbiFgF5rYNarTo1oKrakXkcfRrRmEZuPPGtShaugbp7HNaIdsaWsnNMTlqbc2cyyPETCRQCBEjOzDqPEhKoHD+uEe3FpzHBmapk4p8Yl4ovbeXb/12O5ZtR67ZuqlekvWJlJNPmBCxCrcokjBGYbjzMIoqxrUWLG87RnHFuBaL6akl2LX3QLrvsnwuP2c9pumk6ZCxCTEfJFAIEaPINqhJmPUEhFoL41sU4W6nA+fUEdjzMj+5ryGS7lsyuYr5JoPZQsQqlMQvGesogHHdSrZtY3nbIwPYb+7r42+7Otk3XAi2RXfLfi5771oJEiItpEUhRIwi2V7nmD02zPTUwegggabtgA3+IUxPLbtbvHzjzlcAWOH28ZlSOHV1DsdtrJv2XtawD8OVg5FbML7MIwMEe/YlpbzpMDxYQMA7PPuJGWA+6uKqXDbtXuxzIYFCiFgFkzfrCcCsWALA8MO3Rh1byrZXndlMn/2Hd5ETGIKHH+LkVa4Z7zX84C2YlcsoOOXyccdHtv2CwJ6XklLedMiOEOGYj7q4Vx1Dwan/mvz7Jv2OQmSpZM56AnAt3kDhB26MtFQMdy5jnmW80PgMR62tYdViD7ZdykBeEXYo/fiU5bICWD3NU74W7GnGtXg9ue86Nyllnm9lZYX09Q2luxhJMR91cVUuTcl9JVAIEavQrCeSNEZhGAaumpXjjj33Wisj/mAkE6xhGLMmB7R9XWBbka1TI8etILavE9chW3AvWpuUMs+3guoSBgr7012MpMjkushgthAxCrcojCTNeprKtldbqK0oZPUST+TYrHts+0JBJDCKPdQXOW73d4EdjCQZFCJREiiEiFW4RZGEdRRTaesZ4s1m76TtTE1PHfZA95Rbp4KTmjzyeNICvgPpQoRIlAQKIWIVDI8lpKZF8XRDK6ZhcPzGiWspwnmhph6nsHztYDi/yhMX8AEY0qIQcySBQogYRb7RJ2mMIlrQsnhmRyubVlVSVjw+EB3ICzX1OIXlbcOsWg4u9+QWRW4hRp6svRBzI4PZQsQq4AfTjWEk//vVn5/fi3fAz0lTZIKdaevU8HFX3RoI+LEntChMT924biwhEpHSQKGUuhi4HsgBbtVa3zbh9SOAHwO5wD7gEq11n1JqXeh4Kc7043/VWm9PZVmFmI0dHEva1NhojU29/PdTb3P0upoptzM1cgum3TrVDvixB7qdVkdgdNxmSJa3zQkgQsxRyrqelFKLgZuBE4HDgSuUUusnnPY94Eat9WZAA9eGjv8U+A+t9eHAF4FfpqqcQsQsMJq0xXZhvf2j/Pj+HdRVFPLhs9ZO++3f9NRN2aIIj1uYnlpn1zxfhzMtNuDHHuiJtEaEmItUjlGcBjymte7RWg8C9wAXTjjHhdNqACjkwOLFnwF/Dj1uAJalsJxCxMQOjCV1xlMgaPGj+3cwOmbxifMPIz93+gb+xASCYZGZTZ46ZxqsFSDg68LydQL2pCSDQiQilV1Pi4DWqOetwNETzrka+ItS6lZgEDgGQGv9i6hzvgLcF88bV1Yu/MG76uqSdBchabKlLrPVo81lMZZfkLT6PvrCXnY1e7nm4iM4fN3Mf9D7Fi2jR2+jstSFmVd44PiuPkaAmlWrGC0yaQXGelopsUcZAiqXH0Jehv98suXzBZlbl1QGChOwo54bgBV+opQqAH4OnKa1fkEpdTXwK+B9odcN4FvAscC743nj7u4BLMue/cQ0qa4uobMzM1doTpQtdYmlHqNDQ9i4klbfB595m7qKQtYv9cx6zzF3OQAdb++O7IgHMNKyF6OglG5fECvUOB/raaW/1weA1y7ByOCfT7Z8vmBh18U0jRm/YKey66kZqI96Xge0RD3fCAxrrV8IPf8xcAqAUsoN3AkcBbxba+1NYTmFiE3An7QxiraeIXZNsbhuOpGZT33ju5+i97AwCjyQk89YTwu2tw2joBQjt3DSvYSIVyoDxaPAqUqpaqVUIXABB8YdAN4CliqlVOj5ecCLoce34IxdnCFBQiwUdsCftFlP2xpaplxcN53w6mrLN35AO3oPi3BeqLGeVixfu4xPiKSJKVAope5VSp0Wz4211vtxZiw9DmwH7gp1MT2olNqite4FPgzcrZRqAD4C/LNSqhr4N0ABzyultiulZGqsSL/AGEYS9qIIWhbPvtbGplWVeIpjW+VtuHOdrVOjWhS2fxh72IsRNbPJ9IQCRV8bhqTuEEkS6xjFfwM3KKVuB34C3KG17pntIq31XcBdE46dHfX4IeChOZRLiHljB/1JWZX92u4evIN+tm6qn/3kKGZZ3bgWRfhxdMvB9NTi3/MS2BZmmQQKkRwxtSi01ndqrU8GzgVqgBeVUr9WSk2cxSRE9krSGMW2hhZKi3I5bNXkxXUzcdKNRwWKvvDU2OgWRR3YVuR8IZIh5jEKpZQJrAbW4Hzj7wBuV0r9e4rKJsSCYgf8c94G1TswyqtvdXPCxjrcrviGCMNbp9ojA0BUi6J0fNdT5LEkAxRJEusYxVdxUmz8P+D3wKFa62uAk3HGE4TIfsG5tyhefrMTy7Y5/rD4up0gOueT05Kw+towiirGlWlcN1RpzZzKKkRYrGMBNcBZWuuG6INa60Gl1D8mv1hCpIc12IuRVzQpINiWBcHAnGc9NTb1Ulmax6LK+KethoNAYO+r2MEAVve+Sa0GI78Ys6AY28xN6QZL4uASa9v3K8DHAZTjPqVUHYDW+i+pKpwQ88m2LYbuvRH/9j9NfjEY3rQo8UBh2TY7m3pZu7w8oYyuRmkVuPPw/+2PDP/pG1i9zZjliyedl1t7CGaK9k4WB6dYWxS/AP439LgJeAK4Azh7mvOFyDj2YC/2SD/Brr2TX4tsWpR4oGjuGGBwJMC65eUJXW+Yboou+Heswd7IMVf1IZPOqz3/Grp7BhMupxATxRooqrTW3wfQWo8AtyqlLktdsYSYf+EZRVOl8yYJmxbtbHL+wK9dlliggFDyv1kW0rkKSzAkTogkirXrya2UiuyoopSqxcndJETWiAwS93diW8HxLwbm3qJobOqltryAitL8hO8hRDrE2qL4DrBdKfVnnER/pwGfTVmphEiDyBoFK4g90I0RNWvowDaoiQWKoGXxZnMfx6yTtQ0i88S64O4O4HTgb8BLwJmhVddCZA3L2wahQeZJez+EWxQJDmY3tQ0wPBpkbYLjE0KkUzwrfvbhbD50PzColDo9NUUSIj1sbzuu2tXA5P2p7fCspwRbFI1NTsabuYxPCJEusS64+wrQDrwN7MTJ/PqdFJZLiHllW0EsXyeuutWQkz9Fi2IUSHyMYmdTL4uriygtSv6e20KkWqwtig/hbEd6D04Kjw8Dr6eoTELMu4C3E+xgZFbRpBZFIPF1FIGgxa5mL+ukNSEyVKyBokNr3Qo0Apu11r8GDktdsYSYX2Pdzp5ahqc2tD/1+EDBHNZRvN3iwx+wEl4/IUS6xRooxpRSqwANbA3tQCdz/ETWGOt1tncPtyjsga4D4xJEz3qKfx3F9l1duEyDNcvKklJWIeZbrIHiazj7UPwJ+Ducge3HUlUoIebbWHcL5BRg5Jc4yfdsG8vXeeCEBNdRBIIWz+5oZdOqSory576XhRDpEOs6CrfW+lQApdThOOnGG2a+RIjMMdbbillW52wnGlr5bHvbodxZZ5roOoqG3d34hsbYunnR7CcLsUDFGii+hjMtFq31EPBqykokRBqMdbdiVq0EJqfzBpwWhWFimPFtvvh0Qyue4lwOW1mRtLIKMd9i/dS/ppT6IrANGAgf1Fq/kpJSCTGP7OAYAW8nuauOA3DSjOcVjxvQtoNjcbcm+gZGadjdzXuPWYbLjG+TIiEWklgDxTGh/y6POmYDK5NeIiHmmeXrAOxxu8MZZXUTWhSjcY9PPLujDcu2494bW4iFJqZAobWenMtYiCwRbjmM3x2ulmDLG5HndmAMXLEPRtu2zbaGVtYs8VBbEf8mRUIsJDEFCqXU1VMd11rL6myR8cJpxcftN+2pJbDrGeyxUYycvLi3Qd3V7KW9Z4hzjluX9PIKMd9i7XqKXlyXi7NX9v8lvzhCzD/L245ZWIqRVxQ5Fm5dWL52XJXLnFlPcQSKFxs7yM0x2aJk32qR+WLtevrn6OehvSl+npISCTHPLG8bORXjxxGiZz65KpdBwB9X5tjGvb2sWVJGXq4rqWUVIh0SmoqhtW4BViS3KEKkltXfiW3bk49726cNFMH9jQRaNfZwf8wtCu/AKC1dg5KyQ2SNRMYoDGAL0JGSEgmRAlZfK4N3f4GCsz6De+mmyHF7bBR7qI+c8noCUecbOfkYJdWMNT7OWOPjALgrlsT0Xjv39gHI3hMiayQyRmEDe5Ed7kQGCXbvA2yCXXvHBYrwFNicykXjAgVA4TmfC02ddbiqlsf0Xo1NvRTkuVleWzLXYguxIMQ8RqGUOklr/ZRSqgI4SWvdnOKyCZE0kf2wJ2SFtXzO85yKRQxPuMYsqcIsqYr7vXY29aKWlmGasq28yA6xblz0VeDfQ08LgeuUUtenrFRCJFk4QNgTNiSy+kItivK6Sdckots7QkffsIxPiKwS62D2B4AzAEItiZOBf0hVoYRItnDLIfxv9HGjqBwzNzlZ83fu7QVkfEJkl1gDRY7WeizquR+wUlAeIVLC7msDw8Ae9mH7hyLHLW87ZmntDFfGp7Gpl+KCHBZXF81+shAZItbB7GeUUnfirJ2wgcuA51NWKiGSyB4ZwB4dwFW3hmDbm1jedlzVTlYa29uOa8WRyXkf26axqZe1y8sxDRmfENkj1hbFlUAb8F3gltDjT6WqUEIkU7i7ybXMme0UHti2RwexR/rHpe6Yi46+YXr7R1knO9mJLBNToNBaDwL3a603A6cDz4X2pRBiwQsPZLuXHAYYkefhf40kBYrGJhmfENkp1llPNyOznkSGsrzO+IRZvjFEa8cAABiUSURBVAijuCJqqmw4GWByZjy93eKjpDCHOskWK7JMrF1P5yGznkSGsrztGMVVGK4cTE/d+BaFYWCWViflfbr6hqktL8SQ8QmRZWTWk8h6lrctMg5hemqxvO3Ytj0ugCRDl3eEqrLkTLMVYiFJdNbTh5FZTyIDhANCTu2hQCjZn38Ie6R/XACZq6Bl0eMbpcojgUJkn3hnPX0H+BbQClyVqkIJkSz2sA/GRiLjEJF9JrztzhqKJAWKHt8olm1T5SlIyv2EWEhibVFsAlYDPTjZY48HNLBspouUUhcD1wM5wK1a69smvH4E8GOczZD2AZdorfuUUmXAnTh7cncCF2mtx+deECIG1oTd6yLpw9t2wdhw0gayu/qcTFHV0qIQWSjWFsXPgGeAEuA3gBe4d6YLlFKLgZuBE4HDgSuUUusnnPY94MbQtFsNXBs6/lVgm9Z6HfDT0HlCxM2esB+2UVIFhovgvldDx5PTouj0jgBQVSYtCpF9Yg0Uttb6P4AngJ3ARYRmQc3gNOAxrXVPaB3GPcCFE85xAaWhx4UQSeD5PpwWBcBvgbOUUskZcRQHFcvbBqYLo7gSAMN0Y5RWOS0Kkjc1tss7jGkYVJTmJeV+QiwksXY99Yf+3Q1s1Fo/o5QKznLNIpyxjLBW4OgJ51wN/EUpdSswCBwz8VqtdUAp5QOqgZZYCltZWRzLaWlVXZ09exUs5Lq0jXRjl9dRU3tgtXSweglD3nYw3dQcsgLDdLYrnUs9+oeDVJUXUFfrmWuRk2Ih/0ziJXVJv1gDxfNKqd8DNwAPKKXWwKR9XiYycWZIhRlETalVShXgzKI6TWv9QmgXvV/htCYmTkQfd+1sursHsKzJW14uFNXVJXR29s9+YgZY6HUZ6diPUVI9roxj+U7rwiypoqvbSTAw13o0d/ioKM5dEP8vFvrPJB5Sl/lhmsaMX7Bj7Xr6DPBdrfWbwKdD1/3jLNc0A9EbEdcxvkWwERjWWr8Qev5j4JTQ4/2h81FKuXHGRrpjLKsQANi2heVrxywb370Ufm4kqdsJwmsoZHxCZKdYd7izgedCjx8AHojhskeBLyulqnG6lS4Aroh6/S1gqVJKaa01zurvF0OvPQh8CPga8Pc4A9vRC/6EmJU92AvBsUlpxMPPkzWQ7R8L4h3wy4wnkbVibVHETWu9H/gi8DiwHbgr1MX0oFJqi9a6F2fh3t1KqQbgI8A/hy6/AThWKfU68Angk6kqp8heVmTG04RAUbEYDBeuyhlnd8es2xea8SRrKESWinWMIiFa67uAuyYcOzvq8UPAQ1Nc1wOcm8qyiew3XdI/s7CMoou+hlGSnBxPnX3hqbHSohDZKaWBQoh0srzt4MrFKJq8P0Syup3AmRoL0qIQ2StlXU9CpFs4l5NhpPZj3tU3gttl4inOTen7CJEuEihE1kpmLqeZdHqHqfLky/anImtJoBBZybaC2L7OpK28nklXn6QXF9lNAoXISnZ/F9jBeWlRdHmHqZbxCZHFJFCIrHRgP+zUtiiGRwMMjgSkRSGymgQKkZUmphdPlc5IenFpUYjsJYFCZCXL2w65BRj5qU3C1hVKL14pq7JFFpNAIbKSMzW2DiPFM5EiGxZJnieRxSRQiKxk+eZrauwI+bkuivJl7arIXhIoRNaxA37s/u5JyQBTYX/nALUVhSlvuQiRThIoRNax+jsBe1J68WTzjwV5a78PtXRyihAhsokECpF1IjOeUtyi2L3fSyBosW55eUrfR4h0k0Ahso49TXrxZGvc24dpGKyRFoXIchIoRNaxvG0Y+SUYeUUpfZ+dTb2sqC+hIE8GskV2k0Ahso7lbcdIcWtieDTAnlafdDuJg4IECpF1nKyxqR3I3tXsJWjZrJVAIQ4CEihEVrHHRrCH+lI+PrFzby9ul8Ghiz0pfR8hFgLpXM0C1pAXI68Qw5WT7qKkhW1ZWF3vgBXE8nUAk7c/TbbGpl5WLfKQl+NK6fsIsRBIoMhwtm0xdM/15Gw4jbwjz0t3cdIisOsZRp78+bhjZsXilL3f4MgYe9v6OffEQ1L2HkIsJBIoMpw92Is90o/V3ZTuoqRNsKcZ3LkUnHEVAEZuIa6yRSl7P723DxtkIFscNCRQZDirz1lcFl5kdjAK743tXrJxXt5vZ1MvuW6TlYtK5+X9hEg3GczOcJavPfRvB7Zlpbk06TEfs5yi7W7xsXJRKW6X/PqIg4N80jNcuEVBMIA92J3ewqRBZG/seUgACGDbNm09Q9RXpXYxnxALiQSKDGf52iGUuTS8/efBJLI3dooTAIb5Bv0Mjwaoryicl/cTYiGQQJHhLG8brppDI48zyY63u2lq65/TPeYrAWBYW88QAHWVEijEwUMCRQazrQC2rwtXvQJ3Xka1KCzL5kf3v84tv/sb3aHtRBO6T6jOqU7ZEdba7QSK+grpehIHDwkUGSzS7eKpxfTUZlSg2NvRz9BogMGRALfft4OxQGID8Za3DXILU743dlhbzxC5bpPy0rx5eT8hFgIJFBnswLfpuowLFI1NvQBccsYa9rT6+N1juxK6jzPjqXbedphr6xmitqIQU3a0EwcRWUeRwSL9855aTE8dgT0vY1sBDHPh/1gbm3qpryzkPUcsobNvmIdf2Mehiz0ctyG+QWnL24arbk3SyvX8G+3s7xqMPD9qbQ1La4ojz1u7BzmkXtZPiIPLwv+LIqZledshtwAjv8RJgmdb2L4ujHmaAZSoQNBi1z4vJxzmlPOCk1exp8XHL/+8k2U1xSyuLp7lDg474Mce6ElaAkDLsvnZn94gaNkYBtg2NLX185mLNgMwFgjS5R2JO5gJkemk6ymDhReaGYYRWXCWCTOf3mntZ3QsGEmB4XaZfPwDG8nPdXPb/+xgeDQQ030sX2hv7CQttvMO+glaNpeeqfj5597Du9+1mDeb+wgEnfGT9t5hbFtmPImDjwSKDBZOXQEHZv1kwjhFY1MPBqCWHciVVFacx8fP3UB77xD/9dBObNue9T6W70DXWzL09Duzr8pLnIHqdcvLGfUHeSc0hbdNZjyJg5R0PWWoA90uzrdpI68Y8ooiKT1SZcQf4JEX9+EPzVIqLMxlaMg/6Ty3y+SkzYsif3SjNTb1srSmmOKC8WnR1y4v58KTV/GHJ3bzf0s8nLZl6YxlSfbe2L2+UQAqQmVWy5y9sHc29XLoYk9kDUVtRUFS3k+ITCGBIkMd6HYJtSgMA7O0NuVdT9vf6uJ/tu3BNAxmmvgTtGy27+riC5ceQY77wJ4NY4Egb+338Z4jpk4D/t5jlrFjTw8PPb+XU49cMuNsJsvbhlFQipGbnK6g3n4nUISDW0lhLkuqi2ls6uWc41fQ2j1EeUke+bnyayMOLvKJz1AHul0O9M+bnlqCbW+m9H3buocwDPjhNSeR43ZRXV1CZ+fk1dXbd3Xx/XsbuPORXXz4rLWR42/t9xEIWtNuIWoYBketreFXD2s6eoepnSFVRrKTAfb2j5LjNse1dNYtL+eJ7fsZC1i09QxRJ6k7xEFIxigylNU3udvF9NRhD3RjByZ3BSVLa/cQVZ78ca2EqRy+uor3Hbecp15t4emG1sjxxqZeTMNALS2b9trwIHd4rcV0LG87RhJTd/T0j1BekjeuFbNueTljAYvd+7209QxSLwPZ4iAkLYoMZfsmd7uEg4bl68BVsSQl79vWM0R9ZWyDuR/Yegi793v59V80LV2DGCa8vLOTFfUlFORN/9GrKS+gvCSPxqZeTnnX1F1Ukb2xy5IXKHr7RyPjE2FrlpZhGPB8YzvDo0FpUYiDkrQoMpTlbZ+UCC/VU2Qt26Y9ju4Xl2nysfM2UltewKMvN/PIi830DoxyzPqZ/7gbhsHaZeXs3NuLNc3sp/DsrmQmA+zxjU4afC/Md7OiroTnXnfeT6bGioNRSlsUSqmLgeuBHOBWrfVtUa8dDvwi6vRqoFdrvVEptQL4FVAK9AGXaa1TvtenNdCNPThzd0cyjPiLCPYOzn7iDKy+VlxLN487Fm5RBFt2YhZO37WTqL6BUertdlbl5hFsd7pnZqtLMfClcyonHB0l2P7WjO91ZEU/LTtbaNc7qCmfPMso0LoTIGnpxS3Lpm9glPKS/EmvrV1ezp5WZxxGpsaKg1HKAoVSajFwM3AkMAo8q5R6XGv9BoDWejtweOjcQuAF4OOhy28Cfqu1/qFS6srQfS5JVVnDBu/+AgRGU/02DCXpPq6K8ftCG7kFGMWVjL3+KGOvP5qkdzkgF7jaAzTCUKNzLFl1mWh1+L2eemj693C5MUtrkvJ+3oFRgpY95XTedcvLeei5vZIMUBy0UtmiOA14TGvdA6CUuge4EPjKFOd+HnhSa/106LkLpzUBUAQMp7CcEYXn34g9kPpd4jyeQrzeOf6JNVy46lZPOlx4zudS1vX0yptdPLF9Px97/3qKQjODklKXafz8gUaqPfmce+IhU75uFJVjuJPzh7vL63zEJo5RAKxeXIbLNKiTZIDiIJXKQLEIaI163gocPfEkpZQHuAI4LOrwDTgtkKtwvsgeF88bV1bGlitokuq1s5+TJCnr6a4uAVal5NZ797zKPpebVcefPG5mUKrq4nmriMdf3c/lhx+Py0ztH+i3XnM+qiuXV1BdPTll+buPXEp1ecGUry1EmVLOWEhd0i+VgcIEokciDWCqTQcuAe7TWndEHfslcIXW+n6l1AXA/yilNmmtZ8/rAHR3D2BZMZ2aFtOtPVjo9uz3UlNeSFfXQORYKuuyoqaIv4wEeHlHS8oztnaHWhRGIDhlfS4+1dlFMBN+bpn6+ZqK1GV+mKYx4xfsVM56agbqo57XAS1TnPcB4HfhJ0qpamCt1vp+AK31vaFrq1JXVBELZ2rs/M36CS/K2znLeopk6Oobxu0yKC7Mmf1kIQ4yqQwUjwKnKqWqQ4PVFwB/jj5BKWXgDHb/NepwFzCilNoaOucEoF9r3ZnCsopZDI8G6O0fndd1BGXFedRXFs668C4ZuvpGKCvOkzEIIaaQskChtd4PfBF4HNgO3KW1fkEp9aBSakvotGrAr7UeibrOBv4OuEUp1QB8EyfIiDRq7w1lTp3ndQQbD6lk595eBobHUvo+Xd7hKQeyhRApXkehtb4LuGvCsbOjHnfgdCtNvO4F4JhUlk3EJ5xie75XJp9wWB2PvLSP515vmzWb7Fx0e4dZXpuZA41CpJqszBYxaQ0lA6wpn99Asay2hOW1JWxraI1pj4pE2LZNV9+ItCiEmIYECjGlzr5hHnulOZJCo61niGpPATnu+f/IbN1cz76OAZraUzNjpH94jEDQmnKxnRBCAoWYwog/wK1/eJXf/OVN/vTsO4ATKNKV5+iY9bW4XSbbGlpnPzkB4Q2LpkrfIYSQQCEmsG2bXzy0k7aeIdYs8XD/tj3seLs7rmSAyVaUn8MWVc1zr7fjHwsm/f7hLVArJD2HEFOSQCHG+b+Xm3mhsYPzt67kMxcdzqKqIm67bwf+gJXWzKknbqpneDTAK28mf5b0xJ3thBDjyX4UB4mG3d3seHvmPFZB2+ap7S1sXlXJ2cctxzQMPnH+Rr7yy5cAqE/jXgxrl5dT5clnW0Mrx25I3q524AQKl2lQWpib1PsKkS0kUBwEAkGLOx5sZGgkQO4sg9Er6kq4/P3rIwvP6iuL+Jdz1nPftrdZWpO+6aOmYXDshjoe+Os7DI2MUZifvBXUPb5RKj35mCnOJyVEppJAcRB47e1ufIN+rrzgMN61ujru649YU80Ra+K/Ltk2rCjnT8++g97Xl1A9ptPbP0KlZ/KeF0IIh4xRHAS2vdqKpyiXTasmbiCUWVYu8pDrNpOe0qO3f5SqMgkUQkxHAkWW8w6M0rC7m+M31uEyM/vHneM2OXSJJ6lJAm3bprff6XoSQkwts/9yiFk9u6MNy7Y5cVP97CdngHXLy2nuHMQ36E/K/fa09uMPWCyvk/QdQkxHAkUWs22bpxpaWb3EQ31lduz1HEk9vjc5rYptDS3k5pgcv2nR7CcLcZCSQJHF3trvpb1nKGtaE+DMysrPdbFzb9+c7zXqD/L8G+0cpWqSOotKiGwjs56i/Pn5vXR6U789d0F+DsMjqU2bDbCnxUderouj1tak/L3mi8s0UUvLxg1od3mHeWlnJ6dtWYLbdeC7z+DIGM80tHLyuxaTl+OadK+XdAcj/mBWBVIhUkECRZSXdAcdvakPFKZpzNtWrWdsWUp+bnb9mNcuL+fV3d30+EYoKsjh+/c00Nw5SN/AKP9w6moALNvmp398g4bd3ZimMWWK8m0NrdSUF7Bmadl8V0GIjJJdf0Hm6PoPbZn9pCRYyHvnZoJ1oXGKxqZeGpt62d85yPoV5fzlxX0cutjDlrU1PPDXJhp2d5Of62JbQyunHrkEI2r3uvaeId7c18cFJ68cd1wIMZmMUYiMs6SmmKJ8N/+z7W2e3dHG+09Ywacu3Mwh9aXc8WAjj/9tP/dte5tj19dywcmr2NcxwN72gXH3ePq1VgwDjt8o3U5CzEYChcg4pmGwdnk5Pb5RNh5SwbknHEKO2+QTH9iI22Xy64c19ZVFXPbetRy7wUlR/lRDS+T6QNDimdda2bSyUhIBChEDCRQiIx2/sY5DF3v4l/evj+RoqvTk87HzNnBIfSmfPH8jebmuSIry56NSlN/92Fv0Dfg59cgl6ayCEBlDxihERnrX6uop8z1tWFHBhhUV446duKme595o55U3OzFNg0dfbub0LUvZuDKzU5oIMV8kUIisF05R/uBzTXT2jXDoYg8ffPeqdBdLiIwhXU8i65mGwYmb6mnuHCQ3x+RfQ2MZQojYSItCHBRO2ryIxnd6Oe/EQ2QAW4g4SaAQB4Wy4jw+909HpLsYQmQkaX8LIYSYkQQKIYQQM5JAIYQQYkYSKIQQQsxIAoUQQogZSaAQQggxIwkUQgghZpRt6yhcQCRJ3EKWCWWMVbbUJVvqAVKXhWqh1iWqXJO3ggQM256fndbmyYnAtnQXQgghMtRW4OmJB7MtUOQBRwGtQDDNZRFCiEzhAuqBF4HRiS9mW6AQQgiRZDKYLYQQYkYSKIQQQsxIAoUQQogZSaAQQggxIwkUQgghZiSBQgghxIwkUAghhJiRBAohhBAzyrZcTxlNKbUCuB7oA97WWt+e3hIlTinlAh4FPqu1find5UmUUuok4HLAAJ7UWv8szUWKm1KqCvgeMAg8oLW+P81FSlg2/DzCMul3RFoUC8s1QBNQCjyb5rLM1ReAlnQXIgnKgI8BlwHnpbksiboKuFVrfQVwRboLM0fZ8PMIy5jfEWlRpJFS6rPAmVGHLOBHwDvAncAH0lCsuE1Rjx8CrzNNJsqFbIq6nIHz7fUbON/KM1Ed0JzuQiSD1vp/Q9/EM/nngVLqAjLod0QCRRpprb8FfCv8XCn1C2AAGAL8aSpW3Kaox28BL7AFWAVcmqaixW2KupQB3wVuW+jdAzPYh5PwrTXdBZmrLPl5AFxIBv2OSKBYWMJ/pHqAn6e5LAnTWv8jgFLqy8Cf0luaOfs+sAT4tFJqn9b68+kuUAJ+BnxLKTWG02LNZNnw88i43xHJHpsCSqnwGMM5Wut3QscuxhmozsHpL74tfSWMTbbUA7KrLmHZVKdsqUu21GMiGcxOMqXUMTgbf6yJOrYYuBlnY6XDgSuUUuvTU8LYZEs9ILvqEpZNdcqWumRLPaYigSL5/gX4JONnM5wGPKa17tFaDwL34PRRLmTZUg/IrrqEZVOdsqUu2VKPSWSMIsm01pcDKKWiDy9i/EBiK3D0PBYrbtlSD8iuuoRlU52ypS7ZUo+pSItifphA9GCQgTMVNtNkSz0gu+oSlk11ypa6ZEU9JFDMj2ac6YlhdWTIQpsJsqUekF11CcumOmVLXbKiHtL1ND8eBb6slKrGSaNwAZm5QjZb6gHZVZewbKpTttQlK+ohLYp5oLXeD3wReBzYDtyltX4hvaWKX7bUA7KrLmHZVKdsqUu21EPWUQghhJiRtCiEEELMSAKFEEKIGUmgEEIIMSMJFEIIIWYkgUIIIcSMJFAIIYSYkSy4EyLFQjuyfQq4GOd3Lhf4I3Cj1no0nWUTIhbSohAi9X4IHAecqrU+HDgKUDgbCgmx4MmCOyFSSCm1Amdv5HqttS/qeB1wgtb63nSVTYhYSdeTEKl1JPB6dJAA0Fq3ARIkREaQrichUstCfs9EhpMPsBCp9TywTilVEn1QKbVYKfWAUqogTeUSImYSKIRIIa11C3AncIdSqhQg9O/tQLfWejid5RMiFhIohEi9TwBvAM8qpbbjtDLeAC5Pa6mEiJHMehJCCDEjaVEIIYSYkQQKIYQQM5JAIYQQYkYSKIQQQsxIAoUQQogZSaAQQggxIwkUQgghZiSBQgghxIz+P9IIIQQm8y6xAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.semilogx(10**np.linspace(-7, 5, 100), accuracies)\n",
    "plt.legend(['train', 'validation'])\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('C')\n",
    "plt.savefig('training_and_validation_error_vs_C_with_intercept.png',\n",
    "            dpi=300,\n",
    "            bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cross Validation for C Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using `LogisticRegressionCV` lets us automatically select the best C. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "         steps=[('scale',\n",
       "                 StandardScaler(copy=True, with_mean=True, with_std=True)),\n",
       "                ('poly',\n",
       "                 PolynomialFeatures(degree=2, include_bias=True,\n",
       "                                    interaction_only=False, order='C')),\n",
       "                ('model',\n",
       "                 LogisticRegressionCV(Cs=array([1.00000000e-07, 1.32194115e-07, 1.74752840e-07, 2.31012970e-07,\n",
       "       3.05385551e-07, 4.03701726e-07, 5.33669923e-07, 7.05480231e-07,\n",
       "       9.326033...\n",
       "       1.41747416e+04, 1.87381742e+04, 2.47707636e+04, 3.27454916e+04,\n",
       "       4.32876128e+04, 5.72236766e+04, 7.56463328e+04, 1.00000000e+05]),\n",
       "                                      class_weight=None, cv=3, dual=False,\n",
       "                                      fit_intercept=False,\n",
       "                                      intercept_scaling=1.0, l1_ratios=None,\n",
       "                                      max_iter=100, multi_class='warn',\n",
       "                                      n_jobs=None, penalty='l2',\n",
       "                                      random_state=None, refit=True,\n",
       "                                      scoring=None, solver='lbfgs', tol=0.0001,\n",
       "                                      verbose=0))],\n",
       "         verbose=False)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_model = Pipeline([\n",
    "    ('scale', StandardScaler()),\n",
    "    ('poly', PolynomialFeatures(degree=2)),\n",
    "    ('model',\n",
    "     LogisticRegressionCV(fit_intercept=False,\n",
    "                          Cs=10**np.linspace(-7, 5, 100),\n",
    "                          cv=3))  # 交叉验证数为3\n",
    "])\n",
    "cv_model.fit(p3_df[[\"fare\", \"age\", \"pclass\", \"sex\"]], p3_df[\"survived\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([30.53855509])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_model.named_steps[\"model\"].C_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In HW6, there is an optional exercise that shows an alternate better technique for picking optimal C."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cross Entropy Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "delta_y = np.linspace(0, 1, 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEQCAYAAACwSgOGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3yV9d3/8VdCdkggCYFA2EK+7C1O3KPiAAVHXVWr3rZa22rvPtrb/lrr3fbuo8vRWmvVDttSBxXrwC0qVRFRSJhfNiQQIJMMss/1++McbEwhOQm5znXG+/l45MG51jmfLyc573Ot7zfOcRxERETivS5ARETCgwJBREQABYKIiAQoEEREBFAgiIhIQILXBfRQMnA8UAq0eVyLiEik6AMMBj4GmjoujNRAOB5Y7nURIiIRag7wr44zIzUQSgGqqurx+bp/H0VOTl8qKup6vahwpjbHBrU5NvS0zfHxcWRlpUPgM7SjSA2ENgCfz+lRIBzeNtaozbFBbY4Nx9jmIx5q10llEREBFAgiIhKgQBARESAE5xCMMZnAB8BF1tqdHZZNAx4HMoH3gNusta1u1yQiIv/J1T0EY8wJ+C9tKjjKKn8F7rDWFgBxwC1u1iMiIkfn9iGjW4Dbgb0dFxhjRgCp1toVgVl/Ai53uR4RETkKVwPBWnuztfZoN5AN4fPXwpYCQ92sR0Qkkh2sb+b7T3zE+4X/8R27V3h5H0I80P5C2jjA150nyMnp2+MXz83N6PG2kUptjg1qc3Rqa/PxwOIiDlQ1MHRgX1fa7GUglODvU+OwPI5waKkzFRV1Pbo5Izc3g7Ky2m5vF8nU5tigNkevZ9/ZStHWcr584XhGDM7sUZvj4+M6/SLt2WWn1tpdQKMx5pTArOuAV7yqR0QkXH26uYxXVuzmjGlDOGXy4K436KGQB4IxZqkxZlZg8hrgfmPMJqAv8FCo6xERCWf7qw7xxMsbGJmXwRfPGevqa4XkkJG1dmS7x3PbPS4EZoeiBhGRSNPY3MpvnltLfFwcX50/icSEPq6+nu5UFhEJQ47j8KdXNrG3vJ7b5k1iQP9U119TgSAiEoZe/7iYlRsPsOD045g4Kjskr6lAEBEJMxt3VfHssm3MNLlccMLwkL2uAkFEJIxU1jTyyPPrGJSdyk1zxxMXFxey11YgiIiEiZbWNh5espbWNh93XDaZ1OTQ3iqmQBARCRN/e2MzO0prueWiCQzOSQ/56ysQRETCwDtr9vBeYSkXnTyS6QW5ntSgQBAR8di2PQf52+ubmTQ6m/mnjvKsDgWCiIiHDtY389vn15GdmcytF08kPj50J5E7UiCIiHiktc3HI8+vo76hhdsvnUzf1ERP61EgiIh45Jm3t7K5uJovXTCO4YO878JbgSAi4oHlhXt585MSzjt+GCdNzPO6HECBICIScltLDvLka5aJI7O4/MzjvC7nMwoEEZEQqqxp5DdL1pLTL4Xb5k+iT3z4fAyHTyUiIlGuuaWNXz+3luaWNu5cMIX0FG9PInekQBARCYHD3Vnv3lfLrZdMZMiA0N+J3BUFgohICLz60W5WbNjPZaePZtqYAV6Xc0QKBBERlxVtK2fxO9uYPX4gc08c4XU5R6VAEBFxUWlFPY++sJ5hg/pyY4i7s+4uBYKIiEsONbbw0D/Wktgnnq9dNoXkRHfHRD5WCgQRERf4fA6/e2E95dUNfPXSyeT0S/G6pC4pEEREXPDsO1tZt72Sa88roGBYf6/LCYoCQUSkl71XuJfXVhZz9oyhnD4t3+tygqZAEBHpRZt2VfGX1yyTRmVz1TljvC6nWxQIIiK9ZH/lIR5espZB2WncNi+8uqUIRmRVKyISpuobW3hgcRFxcXHcuXAKaSkJXpfUbQoEEZFj1Nrm47dL1lFxsIE7LpvMwP6pXpfUIwoEEZFj4DgOf3tjMxt3VfGlL4yLmCuKjkSBICJyDN5YVcK7a/Zy4UkjOGXyYK/LOSYKBBGRHircWs7Tb21hZkEul5422utyjpkCQUSkB0oO1PG7F9YzfFAGN180gfgw7qMoWK6eBjfGXA18D0gEHrDWPtxh+QzgUSAJKAautdZWu1mTiMixOljfzIOLC0lN6sOdC6eQnBTefRQFy7U9BGNMPvBj4FRgGnCrMWZCh9UeBL5vrZ0KWOBbbtUjItIbmlraeGhxEbWHWrhz4RSyMpK9LqnXuHnI6BzgbWttpbW2HlgMLOywTh8gM/A4DWhwsR4RkWPi8zn8/oX17Cyt4b8umcjIvMyuN4ogbh4yGgKUtpsuBWZ3WOcu4HVjzANAPXBCd14gJ6dvj4vLzc3o8baRSm2ODWqzex57fi2rt5Rzy/xJnHeKtyeR3Wizm4EQDzjtpuMA3+EJY0wq8ARwjrV2pTHmLuBJ4MJgX6Ciog6fz+l6xQ5yczMoK6vt9naRTG2ODWqze974uJgXlm/n3FnDOGncQE//n3va5vj4uE6/SLt5yKgEaH9Rbh6wt930JKDBWrsyMP0ocIaL9YiI9Minm8t46q0tzCjI5cqzIqvDuu5wMxDeBM42xuQaY9KABcCr7ZZvBYYZY0xgeh7wsYv1iIh02/a9Nfz+hfWMHJzJLRdPID4+8i8vPRrXAsFauwe4B1gGrAEWBQ4NLTXGzLLWVgE3AM8YY4qAm4Ab3apHRKS7DlQ38ODiQjLTk/j6wvAfAvNYuXofgrV2EbCow7y57R6/ArziZg0iIj1R19DCA88U4vM5fPOKqWSmJ3ldkut0p7KISActrT5+89xaygO9lw7OSfe6pJBQIIiItONzHP6wdCObi6u56cLxmOFZXpcUMgoEEZF2/vHONj7asJ/LThvNiRPyvC4npBQIIiIBr39czCsf7eaM6flceNIIr8sJOQWCiAjw0Yb9PBXoyvracwuIi4LeS7tLgSAiMW/jzkoef2kDBUP7cesl0X2vQWcUCCIS03bvr+XXz60lLzuNry2cQmJCdN9r0BkFgojErLLqBu5/ppDU5AS+ecVU0lMSvS7JUwoEEYlJNYea+dXTa2ht83HXldPIzkzxuiTPKRBEJOY0Nbfx4LNFVNY2cefCKeQPiI0bz7qiQBCRmNLa5uO3z69j574abrtkImOH9ve6pLChQBCRmOE4Dn9+dRNrt1dw3fmG6QW5XpcUVhQIIhITHMfhmWVbeX/tPi45ZSRnTMv3uqSwo0AQkZjw8oe7eG1lMWfPGMq8U0d5XU5YUiCISNRb9mkJz723nRMnDuKL546NybuQg6FAEJGotmLDPv76+mamHpfDTXPHE68wOCoFgohEraJt5Tzx0kYKhvXnK/MnkdBHH3md6fJ/xxgzLxSFiIj0ps3F1Ty8ZB1Dc/ty58IpJEX58Je9IZi4/InrVYiI9KJd+2p5cHEhOZkpfPPKqaQmuzpacNQI5n9prTHmHmA5UHd4prX2U9eqEhHpoX2Vh/jVM2tIS07gW1dNIzMt+sdC7i3BBMIJgZ+b281zgNGuVCQi0kOVNY388qnVANx91XT1T9RNXQaCtVYX7IpI2Kupb+YXT63hUFMr3/7iDPKy07wuKeJ0GQjGmHTg58AFQCLwOvANa22Ny7WJiASlrqGFXzy1msraRu66Yhoj8jK8LikiBXNS+X4gGbgUmIf/cNGv3SxKRCRYdQ0t/PLpNeyrbODOBVMoGKbO6noqqHMI1tqphyeMMbcA690rSUQkOI3NrfzssQ8pOVDH1xZMZsLIbK9LimjB7CEkGGParxcPtLlUj4hIUJpa2nhocRGbi6u5bd5Ephw3wOuSIl4wewhvA08bY36H/3DRV4BlrlYlItKJllYfv3luLXZ3NXddM5OJw/p5XVJUCGYP4Zv4DxH9BPgZYIH/drMoEZGjaW3z8cjz61i/o5IbLhjHGTOGel1S1AhmD+EP1trrgXtdrkVEpFNtPh+PvbiBNVvLuebcAuZMHeJ1SVElmD2EacYYdQ8oIp7yOQ5/eHkTH286wBVnjuHsmdoz6G3B7CHsBdYbY1bw+a4r7uxqQ2PM1cD38N+/8IC19uEOyw3wKJAF7AOustZWBV++iMQCn+Pw5Kub+HD9Pi6dM4ovnDDc65KiUjB7CB8CTwO7gIp2P50yxuQDPwZOBaYBtxpjJrRbHge8APw0cFnrauA73W2AiES3w2HwXmEpF508kotOHul1SVErmD2E4wLnELrrHOBta20lgDFmMbAQuC+wfAZQb619NTD9E0B3lIjIZzqGwaVzRmm0MxcFs4cwtYfnEIYApe2mS4H2B/3GAPuMMU8YYz4FHqHdISkRiW3+MLAKgxAKZg+hlJ6dQ4jHf9/CYXGAr8NrnwGcZq1dZYz5X+BXwA1B1ARATk7fYFf9D7m5sdfXidocG6KhzT6fw2//Uch7hXu58pwCrvnCuE7DIBra3F1utDmYQPgw8NNdJcCcdtN5+E9QH7YP2GKtXRWY/juwuDsvUFFRh8/ndL1iB7m5GZSV1XZ7u0imNseGaGjzv/cM9nLRySM4b2Y+5eVHP3gQDW3urp62OT4+rtMv0sF0f/3DjvOMMcHcI/4mcK8xJheoBxYAt7Zb/gGQa4yZaq0tBC4GPgnieUUkSvkch7+89u8wuHTOaB0mCqGjnkMwxrze7vF3Oyx+nS5Ya/cA9+Dv5mINsMhau9IYs9QYM8ta24C/B9XHjDHrgbOAu3vQBhGJAofD4N01CgOvdLaHkNvu8eXA/7WbDupdstYuAhZ1mDe33eOPgNnBPJeIRC+FQXjoLBA6nhA+2jIRkR7z+Rz++MpG3l+7jwtPUhh4qbNAaP+OKABEpNe1tvl4/KUNrNx4gPlzRnHxySMVBh4Kdg9BRKRXtbT6+N0/17F6SzmXn3kcF5wwwuuSYl5ngTDOGFMUeDym3eM4YLS7ZYlINGtuaePhJetYu72Ca84tUEd1YaKzQLggZFWISMxoam7joX8UsWlXFTdcMI7T1IV12DhqIFhr3w1lISIS/RqaWrn/2UK27TnIzRdN4KRJeV6XJO0Ec6eyiMgxq2to4f5n1rB7fx1fmTeJWeMGel2SdKBAEBHX1Rxq5pdPraG0op7bL53MtLHBdHYgoaZAEBFXVdY08oun1lBR08idC6YwaXSO1yXJURw1EIwx3+9sQ2vtfZ0tFxEprajnl0+voaGplbuvnEbBMA15Es6C6bpiHGCAJUArMA8oOtpGIiIAu/bV8qtn1gDw7S/OYERe7HVRHWk6u8roawDGmLeBGdba8sD0j4B/hqY8EYlEdncVDy4uIj0lgbuvmk5edprXJUkQgjmHMPhwGARUA7o8QESOaM3Wch55fh0D+qVw95XTyM5M8bokCVIwgVBkjPkj8CT+u5S/DHzkalUiEpE+XL+PJ17ayPBBffnmFVPJSEvyuiTphmDGVL4Z/17Bg8AD+EdCu83NokQk8ry5qpjHXtxAwbB+/PcXpysMIlAwI6bVGmP+BxgLrANSAoPbiIjgOA4vvr+T5/+1g+ljB3DbvIkkJvTxuizpgS73EIwxJwLbgJeAIUCxMeZktwsTkfDn8zn89fXNPP+vHZw8KY+vXjpJYRDBgjlk9HPgHKDCWlsCXIf/8JGIxDB/j6VrWbZ6DxecOJybLhxPn/hgPlIkXAXz7qVZazccnrDWLkV3OIvEtLqGFn7x1BrWbCnn6nPGcvkZY4jXwDYRL5gP9hZjTBaBAXOMMcbdkkQknJUfbOD+Zwopq27kK/PVSV00CSYQfgy8C+QZY/4OnAfc6mpVIhKWdu+v5f5nC2lp8XH3lVMxw7O8Lkl6UTCB8CqwETgX6APcZ63d6GpVIhJ2Nu6s5NfPrSU1OYHvXjuD/Ny+XpckvSyYQPjYWjsN2Op2MSISnlZs8N9wlpedxjevmKq7j6NUMCeV640xGvBUJAY5jsPSFbv4/QsbOC6/H9+9dobCIIoFs4eQDuwwxhQDdYdnWmunuFaViHiutc3HX16zLC8qZfb4gXz5wvG6xyDKBRMIX3e9ChEJK4caW3h4yTo27qriopNHMn/OKF1WGgOC6briXWNMNv49hTj8J5bHuF2YiHijrLqBBxcXsb/yEDfNHc+pUwZ7XZKESJeBYIy5D/huYLIVSAI2AJNdrEtEPLBt70F+vbiI1jaHu66cxvgRuqw0lgRzUvl6YDiwGH8HdzcA612sSUQ8sGrTAX62aDXJSX245/qZCoMYFEwgHLDWluK/F2GqtfYvaO9AJGocvpLot8+vY8SgDO65fhaDc9K9Lks8EGzXFccBFphjjHkN0HVnIlGgpdXHX163/EtXEgnB7SH8H/B7/N1fLwCKgWXBPLkx5mpjzAZjzBZjzO2drHehMWZHMM8pIr3jYH0zP39qNf8qKuXik0dy6yUaxyDWBXOV0Uv4wwBjzFRgrLW2sKvtjDH5+PtBmgk0AR8YY5a17zk1sN4g4Bf4r2ASkRDYvb+Wh/5RRN2hFm6bN5HZ4wd5XZKEgWCuMrrrCPPOttb+qotNzwHettZWBrZZDCwE7uuw3uPAD4GfBlWxiByTVZsO8PjLG0hPSeS7185kRF6G1yVJmAjmHEL7E8hJwOnAW0FsNwQobTddCsxuv4Ix5k7gU2BFEM/3H3Jyet65Vm5u7P0RqM2x4Wht9vkcnnrD8vfXLeNGZPE/N84mKyM6Tgfqfe4dwRwyurH9tDFmCPBEEM8dT2AMhYA4wNfueSbhPydxNtCjvpIqKurw+ZyuV+wgNzeDsrLanrxkxFKbY8PR2tzU3MbjL2/gE1vGKZPzuP78cbQ2tlDW2OJBlb1L73Pw4uPjOv0i3e3x7qy1e4GRQaxaArS/xTEP2Ntu+vLA8lXAUmCIMWZ5d+sRkc6VH2zgJ3/9hE83l3HVWWO4ae54EhM01KX8p+6eQ4gDZgEHgnjuN4F7jTG5QD3+vYHPBtax1v4A+EHgNUYC71hr5wRduYh0acPOSn73z/W0+Ry+cflUJo/O8bokCWPdPYfgALuB/+5qI2vtHmPMPfgvUU0CHrfWrjTGLAW+b61d1ZOCRaRrjuPw6srdLH5nG4Nz0rnjssnkZad5XZaEuW6fQ+gOa+0iYFGHeXOPsN5OgjsMJSJdaGhq5Y9LN7LKljFr3EBumjuOlKRgvvtJrAvmkNEyPn9y+HOstWf1akUi0mMlB2r50ZOr2F/ZwBVnjuH82cOIU7fVEqRgvjasAibgv1u5GX9ndwnAUy7WJSLd9Ikt4w9LN5LQJ467r1JPpdJ9wQTCqcCp1to2gEBfRiustf9wtTIRCYrP57Bk+XZe/nAXBcP7c+tFEzTMpfRIMIGQi78zu/rAdAags1MiYaDmUDOPvbCe9TurOH3aEL7+xRlUVx3yuiyJUMEEwiJghTHmOfyXnV4BPOhqVSLSJbu7ikdfWE9dQys3XDCO06YOUed0ckyCucro+8aYNcCZQAPwX9bad12vTESOyOc4vLJiF8+9t52B/VP5xuVTGT4o9rpukN7XaSAYY+KAPtba54wxb+LvsG5vZ9uIiHtqDzXz2EsbWLe9ktnjB/KlL4wjNVmXlErvOOpvkjFmAv4uJe4wxrwFfBRYlGmMucFa+0YoChQRvy0l1fzun+upPdTCdecbzpg2RJeUSq/q7KvFz4F7rLUvGWNuxH/+YAKQj/+SUwWCSAj4HIfXPtrNP97dzoB+KdxznbqsFnd0FgjDrbV/Czw+E3jeWusDio0x/dwvTURqDjXzh5c3UrStglnjBnLjBTpEJO7p7Derrd3jk4E7203rImcRl63bUcETL22kvrGVa84t4KwZ+TpEJK7qLBAqA0NmZuDvpvpdAGPMycCeENQmEpNaWn089942XltZTP6AdO6+chpDB/Z8MCiRYHUWCP+DvwvrfsC3rbX1xphvAfcA80NRnEisKa2o59EX1rN7fx1nzcjnijPHkJSoewskNI4aCNbaFcaYfCDNWlsdmP0BMNtauyUk1YnECMdxWF5UyqI3N5OU0IevLZjM9LG5XpclMabTs1PW2mb8Hdodnv7A9YpEYkxdQwt/fnUTn9gyxo/I4uaLJpCVkex1WRKDdLmCiIfW7ajgj0s3UVPfzOVnHsf5s4cTrxPH4hEFgogHmprbeOadrSz7dA+Dc9K447KZjBqc6XVZEuMUCCIhtrXkII+/vIGyqgbOO34Yl502WieOJSwoEERCpKXVxwvv72Dpil1kZ6Tw7aunY4ZrEBsJHwoEkRAoPlDHYy9uoKSsjjlTBnPV2WN1x7GEHf1Giriotc3Hqx/t5p//2kF6aiJ3LpzCtDEDvC5L5IgUCCIu2bWvlj++spHd++uYNW4g151XQEZaktdliRyVAkGkl7W0tvHC+zt5ZcVuMtISuf3SScw0A70uS6RLCgSRXrS15CB/fGUjpRWHOGVyHledPZb0lESvyxIJigJBpBc0Nrfy3LvbeeuTErIzk7nriqlMGp3jdVki3aJAEDlG63ZU8OSrlvKDjZw1I58Fpx+nK4gkIum3VqSHquuaeOqtLazceIBBWal855oZFAzr73VZIj2mQBDpJp/PYdnqPTz33jZaWn3MO3UUc08cTmKC7jaWyKZAEOmGXftqefK1TeworWXCyCyuPc+Ql53mdVkivUKBIBKEhqZWliz3nzTOSEvi1ksmcML4QRrSUqKKq4FgjLka+B6QCDxgrX24w/J5wA+BOGAHcKO1tsrNmkS6w3EcPt50gKfe2sLBumbOmJHPgtNGk6ZLSSUKuRYIgdHWfgzMBJqAD4wxy6y1GwLLM4FHgOOttXuMMfcB9wJfd6smke7Yvb+WRW9uYXNxNcMH9eWOy6Yweoi6qJbo5eYewjnA29baSgBjzGJgIXBfYHkicLu1dk9gugi4xsV6RIJS19DCkve2886aPaSnJHL9+YbTpg4hPl6HhyS6uRkIQ4DSdtOlwOzDE9baCmAJgDEmFfgO8GsX6xHpVJvPxzur9/L88u00NLVx1oyhzJ8zSncaS8xwMxDiAafddBzg67iSMaYf/mAotNb+uTsvkJPTt8fF5eZm9HjbSKU2H93areX8/vm17CytYcqYAdw6fzIjInQEM73PscGNNrsZCCXAnHbTecDe9isYYwYDrwFvA9/s7gtUVNTh8zldr9hBbm4GZWW13d4ukqnNR1ZaUc/id7axeks5OZkpfHX+JGaaXOLi4iLy/0vvc2zoaZvj4+M6/SLtZiC8CdxrjMkF6oEFwK2HFxpj+gAvAs9Ya3/kYh0i/6Gmvpl/vr+Dd1fvJSkxnktPG835xw/TUJYS01wLhMCVQ/cAy4Ak4HFr7UpjzFLg+8AwYAaQYIxZGNhslbX2ZrdqEmlqaeP1j4t5ZcUumlt8nD59CPNOGUVmusYpEHH1PgRr7SJgUYd5cwMPV+E/zyDiOp/P4YN1+1iyfDtVtU1MHzuAhWccx+CcdK9LEwkbulNZoprjOKzZUs6S5dspKatn1OAMbr14gga3FzkCBYJEJcdxWG0P8McX17OjtIaBWan81yUTOX78QOLV3YTIESkQJOpsKanmuXe3Y4uryc5M5oYLxnHypDwS+ugIpUhnFAgSNXbtq2XJ8u0UbasgMz2JW+ZPYtaYASQmKAhEgqFAkIi3c18NL76/k9VbyklPSWDhGcdx9oyhDM3vH3PXp4scCwWCRKytew7y4vs7Wbu9grTkBC45ZSTnHT+ctBT9Wov0hP5yJKI4joPdXc2LH+xk464q+qYmsuD00Zw1Y6jGMRY5RvoLkojgOA7rd1Ty4gc72VJykMz0JK44cwxnTs8nOUl3F4v0BgWChLXWNh8rN+7n1Y+KKSmrIysjmWvOLWDOlMHqZkKklykQJCwdamzlvcK9vLGqmKraJoYMSOfGueM4cUKerhoScYkCQcJKZU0jb6wq5t01e2lsbmPc8P586QvjmDw6W+MXi7hMgSCecxyHbXtreOuTElZtOoDjwPHjB3L+7GGMzIvMMQlEIpECQTzT3NLGRxv38/Yne9i1v5bU5D6cNWMo584ayoD+qV6XJxJzFAgScuXVDSxbvYf3CvdS39hK/oB0rjvfcNLEQaQk6VdSxCv665OQaPP5WLe9knfX7KVwazlxcXFMLxjA2TOGYob31/kBkTCgQBBXlVU3sLyolPfXllJV20RmWiIXnjyCM6blk52Z4nV5ItKOAkF6XUtrG59uLmd50V427KwiDpg0OoerzxnL1DED1OuoSJhSIEivcByH7aU1rFi3nxUb9lHf2EpOZgrz54zi1MmDtTcgEgEUCHJM9lce4sP1+1ixYT8HqhpI6BPP9LEDOG3qEMaPzNJgNCIRRIEg3VZzqJmPNx7gw/X72L63hjhg3IgsLjxpBDMLBqq3UZEIpb9cCcrBuiY+3VzGKlvGpt1VOA4MG9iXK84cw+zxA3VISCQKKBDkqKpqm/jEHmCVLWNLcTUOkJedxoUnjWT2uIEMHdjX6xJFpBcpEOQzjuOwp7yewq3lrNlazrY9NQDk56ZzyamjmGVyGTIgXfcMiEQpBUKMa2ltY+Ouaoq2lVO4tYKKmkYARgzK4NLTRjPL5DI4J93jKkUkFBQIMcZxHA5UNbBhZyVrt1eyYVclzS0+khLjmTgym4tPGcnk0TlkZSR7XaqIhJgCIQYcrG9mQ3EJK9buZePOSipqmgDIyUzh1MmDmTpmAOOG9ycxQQPOiMQyBUIUOljXxJaSg2wuqWbTripKyuoBSE9JYNzwLOaemMWEkdkMzErV+QAR+YwCIcI5jsO+ykNsKTnIluJqtpQc5EB1AwBJCfEcl9+PBacP4pTpQ8lM6kN8vAJARI5MgRBBHMehqraJnftq2bmvhp2ltezcV0tdQwsAfVMTGTu0H2dMz2fssH6MGJTxWb9BubkZlJXVelm+iIQ5BUKY8vkcDlQ3sKesjuIDdYEQqKWmvhmA+Lg48nPTmTZ2AGPz+zF2WH8G6RCQiBwDBYLHfD6HippGSivq2VNWT0lZPXvK6yitOERLqw+AuDgYkpPO5FHZjBycyci8DIYN7EtSok4Ci0jvcTUQjDFXA98DEoEHrLUPd1g+DXgcyATeA26z1ra6WZMXmlraqKpt4kBVAweqDvn/rW5gf1UD5dUNtPmcz9bNykgmf0A640dkkT+gL/m56QzJSSc5SR/+IuIu1wLBGJMP/F+Rx3oAAAfZSURBVBiYCTQBHxhjlllrN7Rb7a/AzdbaFcaYJ4BbgEfcqqk3OY5DQ1MrtQ0t1B1qobahhdr6Zqpqm6isbaK6ronKmiaqahupb/x8xiUn9WFQ/1SG5aYzsyCXgVmp5GWnkZ+bTnpKokctEpFY5+YewjnA29baSgBjzGJgIXBfYHoEkGqtXRFY/0/AD3E5EHyOw6qN+yndX0Obz8Hnc2hzAv+2ObT5HJpb22hqbqOxxf9v+8eHmlqpC4SAz3GO+BqZ6Ulk9U1mQL8Uxg7rR3ZGMlkZyeT2T2VgVhqZaYk61i8iYcfNQBgClLabLgVmd7F8aHdeICen+52rbS2u5oePr+hyvcSEeFKSEkhN7kNKcgKpSQmkpyWSm51Gv77JZKQlkpmeTGZ60mc//fomk52ZHLY3eOXmZnhdQsipzbFBbe4dbgZCPND+K3Qc4OvG8i5VVNTh8x35W/rR9EvpwxP3nMv+slri4+PoExfn/zf+3/8mJsT3bJjHtjaqqw51f7sQiMXLTtXm2KA2By8+Pq7TL9JuBkIJMKfddB6wt8PywZ0sd83A7DTi2tpC8VIiIhHDzdHO3wTONsbkGmPSgAXAq4cXWmt3AY3GmFMCs64DXnGxHhER6YRrgWCt3QPcAywD1gCLrLUrjTFLjTGzAqtdA9xvjNkE9AUecqseERHpnKv3IVhrFwGLOsyb2+5xIZ8/0SwiIh5x85CRiIhEEAWCiIgACgQREQmI1M7t+gDH1Ld/LI4LoDbHBrU5NvSkze22OeLds3HOUbpfCHOnAsu9LkJEJELNAf7VcWakBkIycDz+7i50h5mISHD64L8h+GP8nY5+TqQGgoiI9DKdVBYREUCBICIiAQoEEREBFAgiIhKgQBAREUCBICIiAQoEEREBIrfriqAYY64GvgckAg9Yax/usHwa8DiQCbwH3GatbQ15ob0oiDbPA36If8jSHcCN1tqqkBfai7pqc7v1LgR+Y60dFcr63BDE+2yAR4EsYB9wVbS/z8aYGfjbnAQUA9daa6tDXmgvMsZkAh8AF1lrd3ZY1uufX1G7h2CMyQd+jL+bi2nArcaYCR1W+ytwh7W2AP8H5C2hrbJ3ddXmwC/XI8CF1tqpQBFwrwel9pog32eMMYOAX+B/nyNaEO9zHPAC8NPA+7wa+I4XtfaWIN/nB4HvB9psgW+FtsreZYw5AX/3EgVHWaXXP7+iNhCAc4C3rbWV1tp6YDGw8PBCY8wIINVauyIw60/A5SGvsnd12mb836xuD4xmB/5AGB7iGntbV20+7HH8e0bRoKs2zwDqrbWHh6z9CXDEvaYIEsz73Af/t2WANKAhhPW54Rbgdo4w1rxbn1/RfMhoCP6+jg4r5fOjsx1p+dAQ1OWmTttsra0AlgAYY1Lxf2v8dSgLdEFX7zPGmDuBT4EVRIeu2jwG2GeMeQKYDmwEvha68lzR5fsM3AW8box5AKgHTghRba6w1t4M4D/69x9c+fyK5j2EeKB9R01xgK8byyNRUG0yxvQDXgYKrbV/DlFtbum0zcaYScAC4H9DXJebunqfE4AzgEestTOA7cCvQladO7p6n1OBJ4BzrLWDgd8CT4a0wtBy5fMrmgOhBH+vfofl8fldr66WR6Iu22SMGYy/6/Ai4ObQleaartp8eWD5KmApMMQYE+ldp3fV5n3AFmvtqsD034n8scu7avMkoMFauzIw/Sj+UIxWrnx+RXMgvAmcbYzJNcak4f+WePiYKtbaXUCjMeaUwKzrgFdCX2av6rTNxpg+wIvAM9bab1hro6Gr267e5x9YawustdOAucBea+0cj2rtLZ22Gf9VKbnGmKmB6YuBT0JcY2/rqs1bgWHm38dX5uHv4jkqufX5FbWBEDhxeg+wDFgDLLLWrjTGLDXGzAqsdg1wvzFmE9AXeMibantHEG2+BP8Jx4XGmDWBn8c9LPmYBfk+R5Wu2mytbQAuBR4zxqwHzgLu9q7iYxdEm6uAG4BnjDFFwE3AjZ4V7BK3P780HoKIiABRvIcgIiLdo0AQERFAgSAiIgEKBBERARQIIiISoEAQERFAgSAiIgEKBJEgGGPuNcbcGynPK9IT0dzbqYjrjDHvAFOBPGttk8fliBwT7SGI9JAxZiQwBX/30pd4W43IsdMegkjPXY9/fImPgC8BzwIYY/oCB4Gh1trSwLxJwBtAgbW21ptyRTqnPQSRnrsef9fSi4EzA8N0Yq2tAzbh70jwsJ8CP1EYSDhTIIj0gDHmVCAdWGatrQTeBq5ut8rHBALBGHMaMAF/H/0YY75qjDnaOLkintEhI5Ge+RLwtLW2LTD9d+DbwP2B6Y/xjwMM8DPg/1lrmwGstb8NZaEiwdIegkg3BYZrvAJ/CBz2T2BMu0FpPgZmGGMWAKnt142CEdskSmkPQaT75gOVQKExJiUwrw3/EJ3X4x+MphD/sIa/BL5irfUBGGMGAAdCXrFIELSHINJ9XwJGAg0dfi4HrjHGJATuSVgL7LTWth/acAr+8axFwo72EES6yVr7ha7WMcYkAQOBOzosUiBI2NIegog7fgC8b61d0WH+ZBQIEqa0hyASnHeCWckYMwP/QPBF+Ae6/xxr7Zd78rwioRDnOI7XNYiISBjQISMREQEUCCIiEqBAEBERQIEgIiIBCgQREQEUCCIiEqBAEBERQIEgIiIB/x+FY3KqrFfHEgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "mse = delta_y**2\n",
    "plt.plot(delta_y, mse)\n",
    "plt.xlabel('$|\\Delta y_i|$')\n",
    "plt.ylabel('Squared Error')\n",
    "plt.savefig('MSE_vs_delta_y.png', dpi=300, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: RuntimeWarning:\n",
      "\n",
      "divide by zero encountered in log\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEQCAYAAAC+z7+sAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZykVX3v8U9V7/taPT0zzDD7D2aGAUaBRFAEEcGwaEA0Jgokbjfikpvc3NyrXlBfmsVENNGgERRjgkpAcEENghghrIIwAwNngFmYpbunu3rfu5b7x1Pd9CzdXd3TTy1Pfd+vV7+6u6qeen6H4vXtM+c5zzmhZDKJiIgEVzjbBYiIiL8U9CIiAaegFxEJOAW9iEjAKehFRAKuONsFHEMZcAbQBsSzXIuISL4oApYCTwBj05/IxaA/A3gw20WIiOSp1wMPTX8gF4O+DaCnZ4hEYv5z/JuaqolGBxe9qFymNhcGtbkwLLTN4XCIhoYqSGXodLkY9HGARCK5oKCfPLbQqM2FQW0uDMfZ5qOGvH0LejN7H3DdtIdWA99xzl03wyEiIuID34LeOXczcDOAmW0C7gZu8Ot8IiJybJmaXnkT8H+dc10ZOp+IiKT4HvRmdgFQ4Zz7D7/PJSIiR8vExdgPAl+c70FNTdULPmEkUrPgY/OV2lwY1ObCsNht9jXozawUOBe4Zr7HRqODC7ryHInU0Nk5MO/j8pnaXBjU5sKw0DaHw6EZO8h+D91sAXY654Z8Po+ISF4bHJngL296mN0H+xb9vf0O+jXAfp/PISKS97r7R+nqG6U9uvj9Yl+HbpxztwO3+3kOEZEgSKR2+ysKL37/W6tXiojkgHjqmmQ4HFr091bQi4jkgMnJJ0UKehGRYJoK+iIFvYhIIMUTGqMXEQk0Dd2IiAScLsaKiAScevQiIgE3NUZfpDF6EZFAevWGKfXoRUQCKa6hGxGRYEvoYqyISLBp1o2ISMBp1o2ISMDpzlgRkYCbmnWjtW5ERIJJQzciIgGnWTciIgEX0xi9iEiwxeMJAIo1Ri8iEkwT8QTFRSFCocUPel83BzezS4HrgSrgXufcx/w8n4hIvorFkhT7sKAZ+NijN7M1wNeAtwFbgK1mdrFf5xMRyWexRMK3oPezR/924PvOuf0AZvZOYNTH84mI5K1YLOHL+Dz4G/TrgHEz+xGwEvgJ8Ckfzycikrdi8fzs0RcDbwDeCAwCPwKuBm5N5+CmpuoFnzgSqVnwsflKbS4ManNwFRUXUV7mRfJit9nPoG8H7nPOdQKY2V3AmaQZ9NHo4NQNBPMRidTQ2Tkw7+PymdpcGNTmYBsaHodU5C2kzeFwaMYOsp9B/xPg22ZWDwwAFwN3+3g+EZG8NRFPUFLszxi9b7NunHOPAX8HPATsAPYC3/LrfCIi+cy7GJt/Y/Q4574JfNPPc4iIBEEskaQk3+bRi4hI+iZiCUqKFfQiIoGloBcRCbjxiTilxUW+vLeCXkQkB4zHEpSWqEcvIhJY6tGLiATchHr0IiLBFYsniCeSlOpirIhIME3EvN2lSjR0IyISTGMTcQDKNHQjIhJMo+Ne0JeX+rNYgYJeRCTLRsdjAJSXaehGRCSQRsbUoxcRCbTJHn2FevQiIsE0qh69iEiwTfXoS9WjFxEJpJFcm3VjZiV+FCIiUqhGx2OEQvi2BMKcfz7M7BzgjXjbAj4InGJm1zrnvu9LRSIiBWZ0LE5FaTGhUPb2jP0C8CjwNiAKbAT+3JdqREQK0Mh4zLc59JBe0Bc55+4D3gzc7ZzbA/hXkYhIgRkd93r0fknnnYvM7Ezg94DPm9lmIK1xejN7AGgBJlIPfdA599iCKhURCajh0RgVZdkN+s8BtwG3OOd2m9lu4GNzHWRmIWADcKJzLnZ8ZYqIBNfgyARNteW+vf+cQe+c+wHwg2kPbXDOTcz0+mks9f1eM2sCvuGc+8oCahQRCbTBkQlWLqn27f39nHXTANwPfARvqOdXZuacc79Ip7CmpoU3OhKpWfCx+UptLgxqczANjcaINFZNtXWx25zO0M0XgE9x+Kyb24FZg9459wjwyOTvZnYL8FYgraCPRgdJJJLpvPQwkUgNnZ0D8z4un6nNhUFtDqbxiTjjE3GKSNLZObDgNofDoRk7yL7NujGzc8zsTdMeCvHqRVkREcEbtgGoqvDvXtS0gn7arJtfzGPWTT3wBTMrN7Ma4GrgroWXKiISPJNBX12e3aA/bNYN8GPgk3Md5Jz7CXAP8FvgSeCbqeEcERFJGZoMeh979GnPujGz4tQ6N+ucc/F03tw59ym88X0RETmGgQwE/Zw9ejNrMbOfAkPAKN7wzTLfKhIRKSB9Q+MA1FWX+naOdIZuvgI8BizBu8v1QeAm3yoSESkgfYPjFIVDvl6MTWd65Qbn3FXTfr/ezJ7zqyARkULSNzRGbVUpYZ9WroT0evQlZjZ1b66ZVQLzn+AuIiJH6Rsap97HYRtIr0f/PeA+M/sWXsD/MXCnr1WJiBSIvsFxX9e5gfRm3XzWzPYDF+HdKHWrc+5mX6sSESkQfYNjrFlW6+s50loX0zn3LeBbk7+b2Yecc1/zrSoRkQIwEYvTPzxBY02Zr+dZ6AaFf7eoVYiIFKBo/xgATXX+Dt0sNOj9uzwsIlIguvpGAGiuq/D1PAsNes26ERE5TtG+UYDsXYw1s60zPBVi4X8gREQkJdo/SjgUor4me9MrZ5tC2bnYhYiIFJpo3ygNNWUUhf3tO88Y9M651b6eWUSkwHX1jdLs84VY0BCMiEjWHOodobleQS8iEkgjYzH6Bsdpbaz0/VwKehGRLGjvHgZgaVOV7+dKZz36fzCzdb5XIiJSQNqjXtBnokefzhII3cC9ZrYL+BpwV7o7TImIyLG1dQ8TDoVoafD3ZilIo0fvnPscsBb4e+AqwJnZZ81sud/FiYgEVXv3MJH6coqL/B9BT+sMzrkkcAA4CJQAG4Ffm9kH5zrWzP7ezG49niJFRILmQOdgRsbnIb0x+j8xs8eAHwHtwGudc1cAZwGfmePYNwFXL0ahIiJBMT4Rp717mJVLqjNyvnTG6N8JfB74sXMuMfmgc67LzD4x00Fm1gh8LnXsqcdbqIhIUBzoGiKZhBUtNRk5Xzpj9BcCDwGXm9llZlY37bnZNiD5OvAJoOe4qxQRCZBXOgYAWJErPXozexvepiPb8f4w3GJmVznnHpjlmPcB+5xz95vZNQsprKlp4f8BIpHM/JXMJWpzYVCbg6FrYJzK8mJOXhshHD561ffFbnM6QzefB97gnNsOU6ta3gzMtLoleMM9S83saaARqDazG51zf5ZuYdHoIInE/FdDjkRq6OwcmPdx+UxtLgxqc3Ds2NXFikg10ejgUc8ttM3hcGjGDnI6QT88GfIAzrmnzGzWBHbOvXny51SP/o3zCXkRkaAan4jzSscgbzlzZcbOmU7Q/8zM/jfwFSAOvBd41swagJBzrtvPAkVEgmRP+wDxRJJ1y+vmfvEiSSfo/wooAv76iMffg7fTVNFsBzvnbgVuXUBtIiKB8/KBPgDWLK/N2DnnDHrnXEkmChERKQQvHeijpaGC2kp/d5WaLp1ZN2HgL4CL8e6KvRf4vHMu5nNtIiKBkkgmeelAH5tXN2X0vOksgfDXwPnAl4EvAq/DW/dGRETm4UDnEAPDE2xc1ZDR86YzRn8R3rIHEwBmdg/wjK9ViYgE0I493tyVk0/MbNCn06MPT4Y8gHNuDJiY5fUiInIMO/b00NpYSWOt/9sHTpdOj/5pM7sRb3plErgO2OZrVSIiAROLJ3D7ejjnlKUZP3c6PfoPAw3Aw8CjQAT4iJ9FiYgEzQuv9DA+kcj4hVhIr0f/f5xz1/hdiIhIkP12ZxdlJUUZvxAL6fXoL/G9ChGRAEskkzz1Yieb1zRSWjLrPaa+SKdHv8vM7sVbqnhqBR7n3Bd9q0pEJEB2t/XTNzjO1vWRrJw/3c3BAVZPe2z+y0qKiBSop3Z2Eg6F2LIu8+PzkF7Q3+2c++H0B8zsPT7VIyISKIlkksd3dLBxVQNV5dlZUWbGoDezS/GWPPhCahmEydXxS4BPA9/xvzwRkfy285Veov1jXPHGtVmrYbYe/Wl4Sx+0AB+d9ngMuNHPokREguLhZ9spLy3i9CyNz8MsQe+c+yzwWTP7U+fcP2ewJhGRQBgbj/OEO8QZJ7VQloXZNpPSGaP/tpldjbcl4NTmhpp1IyIyu0d3tDM2Hs/K3bDTpRX0eDNutvPqbBvNuhERmUUymeT+J/ezoqWa9SdkbjepY0kn6E8FTtb68yIi6du5r5f9nUNcc/FJhEKhuQ/wUTp3xu7zvQoRkYC578n9VJUXc9bGJdkuJa0e/XbgATP7OTAy+aDG6EVEjq2jZ5indnbyljNXZvUi7KR0gr4WeAlYN983N7PPAFfijenfoj8OIlII7nlkL0XhMBeesSLbpQDpbQ5+7ULe2MzOxZuHvwXvJqsdZnaPc84t5P1ERPJBV+8IjzzbzhtPW059dVm2ywFmGaM3s1un/Xz1Ec89PtcbO+f+CzgvdRG3Be+PytCCKxURyQM/fXQvoRBc/Dsrs13KlNkuxm6Z9vPHjngurQUbnHMTZvZpYAdwP3BgfuWJiOSPtugQD25r4/VblmV8u8DZzDZ0E5rhZ5jHPHrn3PVm9rfAj4H3A/+SznFNTdXpnuIokUjNgo/NV2pzYVCbc9vXf7yD0pIirr18Mw01Cw/6xW5zOhdjYQE3SJnZSUC5c+5p59ywmf2Aw/+VMKtodJBEYv73ZUUiNXR2Dsz7uHymNhcGtTm3vbC3h8eea+eKc9cQG52gc3RiQe+z0DaHw6EZO8izBf3x3v26Bvi0mZ2Teq/LgW8e53uKiOScRCLJ93/5Ek21Zbz5tbkx02a62YJ+jZn96Bg/hzh8E5Jjcs791MzOBH4LxIE7nXPfO65qRURy0P1P7mdvxwAfunxTVrYKnMtsQT/9AuydRzx3Rzpv7py7AbhhfiWJiOSPaN8oP/j1LrasbeKMk1qyXc4xzbZM8bczWYiISL5JJpP8272OJEn+6MINWV/TZibprHUjIiLH8NC2Np55Ocrvv34NzXUV2S5nRgp6EZEF6Oge5rb7XuSklfVckCNLHcxEQS8iMk+xeIJ/+fFzFBeFeN8lGwnn6JDNpDmD3syWmNllqZ//1szuN7NT/S9NRCQ3/eC/drG7bYCrLzopp+6AnUk6PfpbgbVmdj5wEfAd4B/9LEpEJFc9/nwHP3/8Fc47fTmvzdFZNkdKJ+ibnHM3AhcDtznnbgUqfa1KRCQH7Ts0yDd/+jzrT6jjDy5Yn+1y0pZO0JeaWQle0N9nZpXAwheiERHJQ31D4/zTnduoLCvmT9+2meKi/LnEmU6lPwQ6gS7n3JPA48BtvlYlIpJDRsdjfPk/nqF/eJzrfn8LdTmyzny65gx659z1wGbgvNRD73bOfdbXqkREckQsnuCmu59LLXGwmTXLarNd0rylNesG2OqcS6aWG77RzNJehVJEJF8lEklu/dkLbN8V5b1vMU5b15ztkhZkobNu/snPokREsi2R9EL+4WfbefvrV3PuacuzXdKCadaNiMgREskk//rzF3hoexuXnb2KS8+ec8HenKZZNyIi0yQSSb79sxf49TNtXPK6VVx+Tn6HPGjWjYjIlIlYnK/etZ0Ht3kh//bXr87ZFSnnY86tBFN7vn7DObc/9dC7nXPbfK5LRCSjhkcn+Mc7t7NzXy9/cMH6nNwpaqHmDHozCwPvNrOLgRLgXjPb4ZyL+V6diEgGRPtG+fIdz9AWHeaDl23irI1Lsl3Sokpnc/C/Bk4Fvow31PMB4AvAn/lYl4hIRuzc18tX79pOLJ7g4+84lU2rG7Nd0qJLJ+gvAl7rnJsAMLN7gGdQ0ItInvv1Mwf5zn86muvK+eiVW1jaVJXtknyRTtCHJ0MewDk3ZmYTsx0gIpLLxififO/+F/nV0wfZtKqBD71tM1XlJdkuyzfpBP3TZnYj8BUgCVwHpHUx1syuB65K/XqPc+4vF1SliMgiaYsOcdPdz7G/c5CLzlrJFeeuoSicPwuULUQ6rfsw0AA8DDwKRICPzHWQmV0AXAicDpwGvMbM3r7wUkVEjs8jz7XzmVt/Q+/gGB+7cgtXnbcu8CEP6fXo/49z7poFvHcb8OfOuXEAM3seWLmA9xEROS4Dw+P8+y928vjzh1h/Qh0fvGxTXuwMtVhCyWRy1heY2Xbn3CnHcxIzWw/8N3C2c+7FOV6+Cth9POcTEZn02LNtfOWOZxgcHuddFxpXnreeojxaS34BVgN7pj+QTo9+l5ndCzwEDE4+6Jz7YjpnNLNNwD3A/0oj5KdEo4MkErP/ETqWSKSGzs6BeR+Xz9TmwqA2z8/Q6ATfve9FHn62nRUt1Xz8yi2sXFJDd/fQIle5uBba5nA4RFPTsVenSSfou1Pfpy/4kFYCm9nZwJ3Ax51z30vnGBGR45FMJnn42XZuf+AlhkZiXPq6VVx69qq82hFqsaWzBMK1kz+bWZlzbiydNzazFcDdwDudc79ceIkiIuk50DnId+7dyc59vaxdXst73mmsXFKT7bKybsagN7NS4BvA3c65u1IP32lmncD701gC4S+AcuCLZjb52Necc187zppFRA4zPBrjJ4/s4RdP7KO8tIhrLj6Jc7YsJRyABckWw2w9+s8AtXgXUSd9EPgqcAPwydne2Dn3MeBjx1mfiMiMYvEE//X0QX740G4GRyY455SlvOO8tdRUlma7tJwyW9BfApzhnBuZfMA5d8DM3gs8whxBLyLil2QyydMvdnH7r16mo3uYk1bWc9X561jVmn/7uWbCbEE/Pj3kJznn+s0srXF6EZHF9sLeHu5+cBc79/fR2ljJR6/YwqnrmgKxbrxfZgv6uJnVOOcOm+djZjV4yxWLiGSMe6WHHz60mxde6aWuupQ/unADbzh1WUHPpknXbEH/XeBmM/tj59wQgJlVATfjTZkUEfHdi/t7ufvB3Ty/t4e6qlL+4E3rOfe0ZZSWFGW7tLwxW9B/Cfga0G5mz+Gti3My8O94F2pFRHyRSCbZ/nKU+25/hud2RamtLOFd56/j3NOXU6aAn7cZg945lwA+YGafA14DJIDHnHNtmSpORApLLJ7g0ec6+M/HX+FA1xDN9RVewJ+2nLJSBfxCpXPD1F5gbwZqEZECNTgywYPbDnLfb/bTMzDGCZEq3n/JRt76hrX05PiSBfkgnSUQRER8sbd9gPuf2s/jOzoYjyU4+cQGrr34JDatbiQUCulC6yJR0ItIRk3EEvzGHeKXT+3n5QP9lJaE+Z1NrZy/dbmWK/CJgl5EMqItOsRD29p4aHsbA8MTLGmo4F1vWs85p7RSGeBt/HKBgl5EfDMyFuPx5zt4aHsbLx/oJxwKceq6Js7bupyNqxq1Fk2GKOhFZFElkkl2vtLLQ9vb+M0LhxiPJVjaVMlV563jdzctoa66LNslFhwFvYgct2QyySsdgzz2fAdPPN9BtH+MirIifndzK+dsWcqapbVaoiCLFPQismDt3cM8tqODx3Z00N49TFE4xKbVjfz+uWvZuiGim5tyhIJeROblUO8IT7lOHtvRwd6OAULAhhX1XHjGCl5jES0RnIMU9CIyq2Qyyb5Dgzy1s5Ondnaxv9PbOnpVaw3vPH8dZ568hIYajbvnMgW9iBwlkUjy0oE+ntrZyW9f7KSzd5QQsO6EOt55/jpO3xChpb4i22VKmhT0IgJ42/Ht2NPNtl1Rtr0cpX9onKJwiI2rGnnr75zIaesj1FVpWCYfKehFClQymeRg1xDbdkXZ/nKUF/f3EU8kqSgrZvPqRk7f0MyWNc1Ulism8p3vn6CZ1QIPA5c45/b4fT4RmdnoeIwX9vamwr2LaL+3WdyKlmouOmslp6xpYu3yWorCWmMmSHwNejM7C/gGsMHP84jIscXiCXYd7GfHnm6e39vDroP9xBNJykqL2LSqkUvPbmLz6kYaa8uzXar4yO8e/fuBDwPf8fk8IoJ3V+qBziGe39PNjr09uFd6GZuIEwJWLa3hLWeuZOOqBjasqNfKkAXE16B3zr0PwMz8PI1IwUomkxyMDvPivl5eeKWHF/b20D88AUBrYyWvO6WVjSc2ctKJ9VRp4bCClbNXWZqaqhd8bCRSeEudqs2FobGxit0H+3l2V5TndnWxY3c3/UPj3nO1ZWw9eQmnrotw6voIkYZgTH8sxM95sducs0EfjQ6SSCTnfVwkUkNn54APFeUutTm4JmJxdrcNsHNfL3s6Bnlud5Sx8TgAkfpyTlnTyIYV9WxYUU9LfcWr68nEYoH471Mon/N0C21zOByasYOcs0EvUmiSySTd/WO8fLCPlw/0s+tgH3s7BojFvQ7PytYaXrepdSrYdTeqpEtBL5Il4xNx9rQPsOtgPy8f6OPlg330DnrDMCXFYVa11nDBa1aw/oQ61q+oZ/XKxoLr3criyEjQO+dWZeI8IrkqkUjS3j3M3vYBdrV5wb7v0CDx1PBkpL6ck1Y2sHZ5HWuW1bKipVqzYmTRqEcvssgSySQdqVDfk/ra2zEwNbZeVlLE6qU1XHTWStYsq2XtsjpqtbSA+EhBL3IcEskknT0j7G7v94K9zQv10VSolxSHWbmkmnNOWcqq1hpWtdawtKmKcFibcEjmKOhF0jQ+EedA1xD7Dg0e9jUyFgOguMgL9d/d3Mqq1hpWt9aytLlSywlI1inoRY6QTCbpGxo/KtDbo8Mkkt6YellJESe0VHHWxiVTPfVlzVUaV5ecpKCXgjYRi9MWHeZA52RPfYB9hwan7i4FaKotY0VLDVs3RFjZUs2KJdVE6isIaw9UyRMKeikIE7EE7d3DHOga5GDXEAc6hzjYNcSh3hFSnXSKi8Isb65iy9pmVrRUs3JJNSe0VGvpAMl7CnoJlFg8QUf3MAe6vCCf/N7RPTI17BIOhWhpqOCESDVnnryE5ZEqljdX0dqk8XQJJgW95KXh0Ql2t/XTFh2ivXuYtqj31dE9PDU3PRSClvoKljVX8RqLsKy5iuXN1bQ2VlJSrECXwqGgl5yVSCTp6h+lPTpMeyrQJ0O9L7WQF3g99EhDBUsbKzl9fXMq0KtY2lRJSXFRFlsgkhsU9JJ1w6MTtHeP0N79au+8vXuYju4RYvHE1Ouqyotpbar0dkFa2UBNWRFLmyqJ1FdotovILBT04rtkMsngyASHekY41DNCR88wh3pHpn4fHHl1hsv03vkpq5tobaqktbGSpU2V1FS+evdoIa5qKLJQCnpZFJNzz6eCvOfVID/UOzJ1UxFACGiqK6eloYLXWoSWhkpaGirUOxfxiYJe0jYRi9PVN0pn7yhdfSPezz0jdPSM0Nk7wthEfOq14VCI5novzNctr6OloWLqq7muQhdDRTJIQS9T4okE3f1jdPWN0tU7QmdfKtB7R+nsG6FvcPyw1xcXhWiu88L7pBPrWZLqmbc0VNBUW66euUiOUNAXkEQySf/Q+FRwT4V5r9c77+4fm5prDt70xMaaMprrKti8upFIXQXN9eU011UQqa+grrpUd4eK5AEFfYCMTcTp7vcCO9o/Snf/aOr72NT36bNYAGqrSonUlbN2eR1nbSwnUl9Bc105zfUVNNaUqVcuEgAK+jyRSCYZGBon2j82FeCT4d0/PE57dPiw2SvgXfSsrymjsbaMVa3eWi1NteVTQd5cV05ZieaZiwSdgj4HJJNJhkZj9AyMpb5G6RkYm7M3XlZSRFNdOa2pG4Qaa8tpqi2jqbacptpy6tUjFxEU9L5LJLxph9MDvGdgjJ7BMXr6U98HxpiIHR7iM/XGm2rLaawto6munMqyYkKhkOaUi8isFPTHYSIWn9YLP+IrFeB9g+OHXeAEb7ZKfXUZDTVeiJ++vpmG6jIaasu97zVl1FWXqjcuIovC16A3s3cDnwRKgC85577q5/kWy9hEnL7BMXoHx+kbGqd3cIzeQS+0+wbH6B0ap3dgjKHR2FHHVpQVUV9dRmNNGUtXNdBQU05DjRfeXpiXUV1RotkqIpIxvgW9mS0HPge8BhgDHjazB5xzO/w652ySySSj4/FUaI9PC3IvwHun/T4yFj/q+KJwiLrqUuqqymipr2D9CfU01HiBXj/5vbqMijL9I0lEcoufqXQB8EvnXDeAmd0BXAl8xq8Tjk3EeeDJfbxysG+qBz71fWiM8YnEUceUFIepry6lrrqMEyJVbFrd6P1eVUZ9TSn1Vd4wSpV64SKSp/wM+mVA27Tf24Az0z24qal63if8+SN7+OodzwBQUVZMY6037n1SczUNteU0pi5kTv7cUFtOVbl3QTPfRSI12S4h49TmwqA2Hz8/gz4MTL8KGQKO7lLPIBodJJFIzv3CabaubeSWT7yZsZExykvnbtrI4Cgjg/M6RU4qxFk3anNhUJvTFw6HZuwg+zmtYz+wdNrvrcBBH89HKBSipbEyrZAXESkUfibifcANZhYBhoArgA/4eD4RETkG33r0zrkDwCeAB4Cngducc4/7dT4RETk2X8c4nHO3Abf5eQ4REZmdbr0UEQk4Bb2ISMAp6EVEAi4X5yEWgTcndKGO59h8pTYXBrW5MCykzdOOOWqTiVAyOb+bkjLgHODBbBchIpKnXg88NP2BXAz6MuAMvCUTjl5dTEREjqUI7ybVJ/AWkpySi0EvIiKLSBdjRUQCTkEvIhJwCnoRkYBT0IuIBJyCXkQk4BT0IiIBp6AXEQm4XFwCIS1m9m7gk0AJ8CXn3FePeP404GagFvg18CHnXCzjhS6iNNp8OfBpvG0bdwPXOud6Ml7oIpqrzdNe93vAV5xzqzNZnx/S+JwN+DrQALQD7wr652xmW/HaXArsA/7IOdeb8UIXkZnVAg8Dlzjn9hzx3KLmV1726M1sOfA5vOUSTgM+YGYbj3jZvwHXOec24AXf+zNb5eKaq82p/2luAn7POXcqsA24IQulLpo0P2fMbAnw93ifc15L43MOAT8C/ib1Of8W+Kts1LpY0vycvwz8v1SbHfAXma1ycZnZWXjLFGyY4SWLml95GfTABcAvnXPdzrkh4A7gysknzexEoMI592jqoVuBd2S8ykz1G6kAAAQmSURBVMU1a5vxekIfTu3sBV7Qr8xwjYttrjZPuhnvXzJBMFebtwJDzrmfp37/PHDMf+XkkXQ+5yK83i1AJTCSwfr88H7gwxxjH20/8itfh26W4a2FM6kNOHOO50/IQF1+mrXNzrkocBeAmVXg9fL+KZMF+mCuzxkz+yjwFPAowTBXm9cB7WZ2C3A68DzwkcyV54s5P2fgfwL3mtmX8PagPitDtfnCOfc+AG8U7iiLnl/52qMPA9MX6QkBiXk8n4/SapOZ1QH3AM84576dodr8MmubzWwz3qbzn81wXX6a63MuBt4I3OSc2wrsAr6Yser8MdfnXAHcAlzgnFsK/DPwrxmtMLMWPb/yNej3463SNqmVw/8JNNfz+WjONpnZUrwlnrcB78tcab6Zq83vSD3/G+CnwDIzy/clrudqczvwonPuN6nfv8vRvd98M1ebNwMjzrnHU79/He+PXVAten7la9DfB7zJzCJmVonXq5scs8Q5txcYNbOzUw+9B/hZ5stcVLO22cyKgB8DtzvnPu6cC8KypHN9ztc75zY4504D3gocdM69Pku1LpZZ24w3SyNiZqemfr8UeDLDNS62udr8ErDCXh3nuBxvKd5A8iO/8jLoUxccPwE8ADwN3Oace9zMfmpmr0297A+BG83sBaAa+MfsVLs40mjzZXgX6q40s6dTXzdnseTjlubnHChztdk5NwK8HfiGmT0HnA/8efYqPn5ptLkHuAa43cy2AX8MXJu1gn3iZ35pPXoRkYDLyx69iIikT0EvIhJwCnoRkYBT0IuIBJyCXkQk4BT0IiIBp6AXEQk4Bb0UPDO7wcxuyJf3FZmvfF29UsR3ZvYr4FSg1Tk3luVyRBZMPXqRYzCzVcAWvGWAL8tuNSLHRz16kWN7L976/o8BVwP/AWBm1UAfcIJzri312GbgF8AG59xAdsoVmZl69CLH9l68JYDvAM5LbVeIc24QeAFvAblJfwN8XiEvuUpBL3IEMzsHqAIecM51A78E3j3tJU+QCnozewOwEW+NdMzsT81spn1ARbJCQzciR7sa+L5zLp76/bvAXwI3pn5/Am+fU4C/Az7lnBsHcM79cyYLFUmHevQi06S2rbsKL9wn/RBYN22zjyeArWZ2BVAx/bUB2OFKAkg9epHDvQ3oBp4xs/LUY3G8rQrfi7fJxzN427v9A/A/nHMJADNrBg5lvGKROahHL3K4q4FVwMgRX+8A/tDMilNz6rcDe5xz07d424K3X69ITlGPXmQa59xFc73GzEqBFuC6I55S0EtOUo9eZP6uB/7bOffoEY+fgoJecpB69CLwq3ReZGZb8Taw3oa3QfdhnHN/spD3FfGbNgcXEQk4Dd2IiAScgl5EJOAU9CIiAaegFxEJOAW9iEjAKehFRAJOQS8iEnAKehGRgPv/S1YXmqrq7p0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cross_entropy = -np.log(1 - delta_y)\n",
    "plt.plot(delta_y, cross_entropy)\n",
    "plt.xlabel('$|\\Delta y_i|$')\n",
    "plt.ylabel('Cross Entropy Loss')\n",
    "plt.savefig('CELoss_vs_delta_y.png', dpi=300, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: RuntimeWarning: divide by zero encountered in log\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAEUCAYAAADN8orUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZykVX3v8U9Xd1fv+9SszD7MD4YBZobFBTCIE1FBwSgaMUJUNF7XJDc3r+SaeyWXlybRKMTdK77EG8XooGKMmhAiCsiLRWCYYZkfA7PvPb1M79XdVXX/eKqH7mamu3qpqq56vu/Xa17ddarqeX7HxvN7znnOc05JKpVCRETCKZLvAEREJH+UBEREQkxJQEQkxJQERERCTElARCTElAREREKsLF8nNrN64CHganffk684RETCLC89ATN7BfAgsDYf5xcRkUC+hoM+AHwEOJSn84uICHkaDnL3mwDMbCpfqwAuAg4DiSyEJSJSjEqBRcBjQHz8m3m7JzANFwEP5DsIEZECdRnBMPwYhZQEDgN0dPSSTE59vaOWllra2npmPai5THUOB9U5HKZb50ikhKamGki3oeMVUhJIACSTqWklgZHvho3qHA6qczjMsM6nHEbXcwIiIiGW156Au6/I5/lFRMJOPQERkRBTEhARCTElARGREFMSEBEJMSUBEZEQUxIQEQkxJQERkRBTEhARCTElARGREFMSEBEJMSUBEZEQUxIQEQkxJQERkRBTEhARCTElARGREMvLfgJmdj3wN0A5cJu7fyUfcYiIhF3OewJmtgT4NHApsAH4oJmty3UcIiKSn+GgzcCv3L3d3XuBu4C35yEOEZHQy8dw0GLG7np/GLg4D3GIiMwJ8aEE3X2DdPcNpf8NjvnZ0z/Emy5dyZqFdbN+7nwkgQiQGvW6BEhm+uWWltppnzgWm/3/Aec61TkcVOe5I5VK0R8f5kTPICd643T1DHKiJ86J3uBnV+9Lr7vSP+ODiVMeq6w0QkNtlIaaCuJDyazUOR9J4ABw2ajXC4FDmX65ra2HZDI1+QfHicXqaG3tnvL3CpnqHA6qc/YlUyl6+4fo6h0MGvG+Qbp6gyv1E+myrt5ButLlw4lTX9dGyyLUVZdTVx2lrjrK/CUNL72uGikvP1lWGS2lpKRkRnWOREomvHjORxK4F7jZzGJAL/A24IN5iENEQiyRTNLTNxQ04n0jDfnQqN9favB7+oZInOLiszRSQn1N0HDX10RZMq8m/XpsYz7SwFdES/NQ04nlPAm4+0Ez+yRwHxAFbnf3R3Mdh4gUn1QqxcBggs6eOCd6BunsidOZHpYJhmdeavB7+oY41ZhCWWmEhpqgUW+qq2D5wjrqa6LU10RpqIlSXx09+bqmsuzklXqhystzAu5+J3BnPs4tIoUnlUrROzDMiZ44nb2DJPd2sP9w18nG/kS6se/sjTM49PKhmGhZJGjEa6PMb6zizCUNJxvykUa9If169BBMGOQlCYiIwMhN1AQdPXE6ugfo6E5fuY9cyffG6ewOruBPNc5eES2lsbaCxpooKxbV0Vg7j8baChpqozTWRGmoraCxNkpVReFfsWeLkoCIZEUylaKnb4iO7nj63wDt3XE6u+PBz57g56lmxtRUltFQW0FDTZS1SxvSjXnQoDfURFm1rJnE4BCVUTVhM6X/BUVkyhLJJCd6Bsc06iNX8iP/OnviDCfGjrpHSkpoqI3SXFfBknk1nLOymea6SprqKmiqq6CxLriqj5ZPfAM1FqsN3YyobFESEJGXGRgcpq0rTtuJAdq7BmhL/2s/Efzs6B4kmRrbwJeXRWiqDRrzNWc0nPy9aVQj31ATJRLRsMxcoiQgEjLJVIqu3sGgYT8xQHtXfNTvQSPfOzA85juRkhKa6ipoaahk7dJGWhoqaa6rpLk+GKZprq8sipkyYaQkIFJkUqkU3f1DHO8coLWzn+Mn+mlN/952YoD27oGXDdNURktpaaikpb6S1UsaaK6vOPm6pb6SxtoKXcEXKSUBkQIUH0pw/ETQsA/saGX3gc50Y99P64mBl91srasuZ15DFSsW1XGBxWhON+5BQ19BdWV5nmoi+aYkIDIHjVzNH2vv52hHH0c7+jne2U9r+qq+q3dwzOejZRFijVXMa6jkrGVNzGusItZQGZQ1VmoWjZyW/ssQyZORB6COtvcFDf2oBv9YRz/98ZfG5UtKoLmuklhjJeetbiE2qpG31fMY6h/UeLxMi5KASJb1DQxz5GRD38exjv6TjX7fuIa+pb6SBc3VrFpcz4KmahY0VbGguZp5DZWUlZ56+4+mukpaB4ZyVR0pMkoCIrMglUrR2TPIobZejrT1nfx5uK2Xzp6Xhm5KgJaGShY0VfGKdQtY0FTF/OagsZ/XUEV5mbb9ltxSEhCZguFEkmMd/RxON/CH2/o40h78HBh1M7aqopRFLcHDUItaaljUXM2C5mpijWroZW5REhA5hWQyRWtnPwdaezl4vCf42drDsY7+MUsKN9VVsKilmkvOXcSiluqgwW+ppqEmqjF6KQhKAhJqI8M4B1t7xjT4h4/3MjgcLFhWAsQaq1gSq2HT2hiLW2pY2FLNwuZqqir0fyEpbPovWEJjaDjJoeO97D3azf6jPexv7eFga8+Yp2MbaqKcEavh8o1LWDKvhjPm17K4pWZObgYiMhuUBKQo9ceH2X+sh71Hu9l3tJt9R3s4dLz35FBORbSUM2I1XHjWfM6I1bJkXg1LYjXUVUfzHLlIbuUtCZjZLUDC3W/OVwxSHHr6h9hzuIu9R7s50jnAzn0dHOvoP/l+fXU5yxbUce6qFpYtqGX5gjpiTVVENGYvkvskYGYNwBeAdwGfzfX5pbANDiXYd7SHXYe72H24i92HujjW+VKDv6C5mqWxWi5Zv5BlC+pYtqCOxlrdpBU5nXz0BK4BdgKfz8O5pYAkkykOHe8NGvvDXew63MWBY70nlzBurq9g5cJ6XrNhMSsX1rF8YR3LlzZrnXmRKShJpU611XL2mdnNAFMYDloB7M5SODIH9A0M4Xs7eG5PO8/tbsf3tdMfD+be11SVc+bSRtYua2Lt0kbOXNZEc31lniMWKSgrgT3jC7PWEzCz64BbxxXvcPfNMzluW1sPyeTUE1csVhe6K8S5Xuf2rgF2HjjBCwdOsPNgJ/uP9ZBKBcsnLI3V8qpzFrJ6SQOrFtUzv6lqzJBOIj5Ea+vLl0qY63XOBtU5HKZb50ikhJaW2tO+n7Uk4O5bgC3ZOr4UllQqxbHOfnbs7cD3dfL8gU7au+IAVJSXsmpxPW9+9QrOPKORVYvrNf9eJEf0/zTJmuMn+tmxt5Md+zp4bm8HHd1Box9sHt7IlRc3cOYZDSydX0tpREspiOSDkoDMmq7eQZ7d085zezvYsa+D1s4BAGqryjlreRNnL2vkrOVNLGyu1mwdkTkib0lAzwcUvkQyyYsHu3h6dxvbd7Wz90gwXlldUYYta2TzhUs5e1kTi2M1mpMvMkepJyBT0t41wNO729m+q41n93TQHx8mUlLC6iX1vPU1q1i/spnlC+q0H61IgVASkAmlUin2He3hiedbeXLncQ609gDB6pkXnRVj/coW1q1o0h61IgVKSUBeZjiRxPd18uTOVra+cJz2rjglJXDmkgbe8do1nLu6hcUtGtcXKQZTTgJmVu7u2suuyAwNJ9i+q53Hdhxj24tt9MeHiZZFOGdlM9deuorz1rRQr8XVRIrOpEnAzC4FLidY5+cB4Fwze6+7/yDLsUmWDSeSPLunnUefO8aTO1vpjyeorSrnAoux8cx5rFvRTEW5llAWKWaZ9AQ+B/wv4FqgDVgH/BBQEihAyWSKHfs6ePS5ozzurfQODFNdUcYFa+dz8br5nLWs6bQbmotI8ckkCZS6+71m9k3gbnffY2a6PCwwRzv6+O32wzz09BHau+JUREvZdOY8Ljp7AetXNqvhFwmpjJKAmV0MXAV8xszWA5oKUgD6BoZ44KlDPLj9MDsPnKCkBNavbOEdr13DhjXziGqoRyT0MkkCnwbuBL7l7rvNbDfwieyGJTOx90g3v3riAI/uOEZ8MMHC5mrefvlqXnXOQprqKvIdnojMIZMmAXf/MfDjUUVrNTto7hkaTvDYjmPc98RBXjzURbQ8wuWblnLR2nmsWlyv6ZwickqaHVTg2rsG+NUTB7n/qUP09A+xoLmad73uTC45d6E2WBGRSWl2UIE60NrDfzyyj4efPUoylWLDmnlcccEZrFvepKt+EcmYZgcVkFQqxfP7O/nlI/vY9mJbMOSzcQmvv2gpscaqfIcnIgVIs4MKxHN7O7j7gV3sPHCC2qpyrr1sJVdsOoPaKv0pRGT6NDtojnt+fyd3P7CLHfs6aayN8u7fX8tl5y3S9E4RmRUZzw4yszIzKwfWuHtiuic0s0sI9h6OEtxjeJ+7753u8YrVniNd3PXrF3l2TwcNNVHetflMLt+wmPIyNf4iMnsymR00H7gDeF36878xsz9y90PTPOf3gLe4+zYzex/wReCaaR6r6LR3DfDj+3fx0NNHqK0q551XrOHyjUu0ho+IZEUmw0FfBh4BrgdKgY8DX2MaDbeZVQB/4+7b0kXbgI9N9TjFKD6Y4JeP7OXfH9lHMpXija9cxlWvXEF1pVb7FpHsKUmlUhN+wMy2uvuGcWXPuPs5MzmxmUWAfwUec/e/zeArK4DdMznnXPXI04f5+k+2c7yzn8s2LOGGN53NwpaafIclIsVlJbBnfGEml5nlZlbp7gMAZlYNTJw5gs9dRzD2P9oOd99sZlHgO+nzfyaDGE5qa+shmZz09C8Ti9XNuQen2rsG+N5/Ps+TO4+zJFbDX717E2uXNkIyOSuxzsU6Z5vqHA6qc+YikRJaWmpP+34mSeBfgHvN7NsEjf/7gB9N9iV33wJsGV9uZrUEPYA24JowLkGRSqW478mDbPn1i6SSKd5++Wpef9FSreQpIjmXyeygW8zsAPAGgnsCd7j77TM453eBF4APuXtyBscpSO1dA3z7F8/xzJ4O1q9s5j1Xmh70EpG8yeiuo7t/G/j2yGsz+5C7f32qJzOzjQQ3lJ8FnjAzgEPu/qapHqsQPfzMEb57z/MMJ5PccKXxexsWa4kHEcmr6U49+Sww5STg7k8CoWv14kMJvnfP8zy4/TCrl9Rz09XrWNBUne+wRESmnQRC15BP19H2Pr7yk6c50NrD1a9ewTWXrqA0orF/EZkbppsEpj49J4SefL6Vb/7bs5RGSvjT687nvNUt+Q5JRGSM0yYBM9t0mrdKAF3KTiCVSvEfj+5ny30vsHxhHR9+63rmNejmr4jMPRP1BCaaBto624EUi+FEku/95/P8ZushLrQY7796nZZ8EJE567RJwN1X5jKQYjA4lOCrdz/NthfbeNMrl/MHv7eKiGb/iMgcpoVpZkl/fJgv3rWN5/d3csOVxuUbl+Q7JBGRSSkJzILegSFu/eFT7DnczQfevI5XnrMw3yGJiGRESWCGBgaHue2HT7HvaDcfeet6Nq6N5TskEZGMTTrLx8w+b2ZrchFMoRkaTvClH21n9+Fu/uQtSgAiUngy6Qm0A/eY2S6Cp4R/MpOdxYpFIpnk6z99huf2dvD+q87mAlMCEJHCM2lPwN0/DawG/hF4B+BmdouZhfrO5w9+9QJP7jzOuzafySXnLsp3OCIi05LRQ1/ungIOAoeAcmAdcL+Z/UkWY5uzfrP1IPf+7gCbLzyD379wab7DERGZtkz2GH4/8EFgPvAN4EJ3bzWzecAz6bLQ8H0dfPee51m/spl3XqFbJSJS2DK5J/BOgt2/fjZ6/X93P25mn8xaZHNQV+8gX//pM8xrrOJD16zXQnAiUvAyuSfweuBB4Boze4uZNYx6byabyxSUZCrF7T9/lt6BYT587XptAC8iRSGT4aBrCTaU2U6QNL5lZu9w9/umc0Izuwy4DYgSbBx/o7t3TOdYuXTPo/t5elc773n9WpbOP/1+nSIihSST8YzPAK9x99e4+6XAlcDnZ3DObwPvcfdzCXYY+x8zOFZOHG7r5cf372LjmfO0HISIFJVMkkCfu28feeHuTzCz/QTOdvdnzawcWALM6V5AMpXijl/uIFoW4YYrTdtBikhRKUmlJm7PzewWoAf4MpAAbgAuAf4UKHH39qme1MzOBe4FhoBXufv+DL62gmD4KKd+/uAuvv6T7XzinRvZfPGyXJ9eRGS2rAT2jC/M5O7mXwGlwN+NK38PQY/glIvlm9l1wK3jine4++Z0z2JB+jmDHwCvziAOANraekgmp94RicXqaG3tntJ3unoH+c4vnmXdiibOW9E45e/n23TqXOhU53BQnTMXiZTQ0nL6+5iTJgF3L5/yWYPvbQG2jC4zs0ozu9bd704XfZeZ3V/Iqrsf3E18MMn1m9dqGEhEilIms4MiwF8AbyR4Wvge4DPuPjyN8w0BXzGz/e7+OMEyFA9O4zhZd6C1h99sPcgVG89g8byafIcjIpIVmdwY/jvgCuCfgC8QDN3843ROll547p3A/zWzrcDbgZumc6xsu+vXL1IVLeOay7TBmogUr0zuCbyBYKmIIQAz+znw1HRP6O4PAhdM9/u5sOdIF9tebOOtr1lFbdW0RsNERApCJj2ByEgCAHD3OMGwTtH62W/3UF1Rxus2nZHvUEREsiqTnsBWM7uVYIpoCvgosC2rUeXRvqPdPLnzONdculJLQ4hI0cukJ/ARoAl4CHgYiAEfy2ZQ+fQfj+6nIlrK5gvVCxCR4pfJpe5fu/sfZzuQuaCrd5DHdhzl985fQk2l7gWISPHLpCdwddajmCPuf+oQw4kUV1yg9YFEJBwy6QnsMrN7CObz94wUuvsXshZVHiSTKe578iDrVjSxqEXPBYhIOGS60TwE606MmMkCcnPSc3s76OiO84evOzPfoYiI5EwmSeBud//p6AIze0+W4smbh585QlVFKRvWtOQ7FBGRnDltEjCzNxMsE/G59NIRI4vnlAN/C/xz9sPLjfhQgt8938rFZ82nvOyU6+GJiBSliXoCGwiWi5gPfHxU+TAvXx20oG3deZz4YIJXnbMw36GIiOTUaZOAu98C3GJmH3b3r+Ywppx73I/RUBtl7bLGfIciIpJTmdwT+I6Z3Qg089KQUNHMDhoaTrJ9dzuvWreAiJaLFpGQySgJEMwM2s5Ls4KKZnaQ7+sgPpjg/DXz8h2KiEjOZZIEzifYF3g6+wfMeVtfOE60LMLZy5vyHYqISM5l8sRwJvv/Fqynd7dz9vImouWaFSQi4ZNJT2A7cJ+Z/TvQP1I403sCZrYReNjdK2ZynJlo7xrgWEc/V2zUMhEiEk6ZJIF64AVgzWyd1MyqgS8B0dk65nT4vs4gnmUaChKRcMpko/n3ZuG8nwduAy7JwrEztmNfB9UVZSydX5vPMERE8ua09wTM7I5Rv9847r1Hp3tCM3sLUO3ud033GLPF93ViyxqJRDQ1VETCaaKewHmjfv8EwVTREZMutm9m1/HyJ4t3EAwvbc40wPFaWqZ/1R6L1Z38vbM7zrHOfq66dNWY8mJTzHU7HdU5HFTn2TFREig5ze+QwXMC7r4F2DK6zMxuAv4auN/MRsq2Ape5e3cmAbe19ZBMTv0xhVisjtbWl06x7cW2oLwuOqa8mIyvcxiozuGgOmcuEimZ8OI50010Z+XhMHe/Hbh95LWZpdx9w2wce6r2HukCYPnC8F1NiIiMmOg5gaJ5KvhU9hzpZkFzNVUV2kxeRMJrohZwlZn96yl+L2HsBjPT5u55uyO750g3tlQLxolIuE2UBD4x6vcfjXsv7zN7ZqKrd5CO7riGgkQk9CZaSvo7p3uv0O1vDbZKXqbnA0Qk5DJZO6joHGnrA2DRPG0oLyLhFsokcLitl6qKMhpq8rpqhYhI3oU0CfSxqKWaEm0iIyIhN2kSMLMF6aUeMLN/MLP/MrPzsx9a9hxp72NRc3W+wxARybtMegJ3AKvN7ArgDcA/A1/MZlDZ1B8fpqM7zsIWJQERkUySQIu73wq8EbjT3e8ACrYFPX5iAID5TQVbBRGRWZNJEoiaWTlBErg3vRdAwc6tbEsngZb6yjxHIiKSf5kkgZ8CrcBxd38ceBS4M6tRZVFbVzoJNCgJiIhMmgTc/VPAeuC16aLr3f2WrEaVRW1dA5SVRqirnnQ1bBGRopfR7CBgk7unzOwfgFvN7LzJvjdXtXcN0FxfQUTTQ0VEpj076EvZDCqb2k4M6H6AiEha6GYHtXUpCYiIjAjV7KDhRJITPYM011fkOxQRkTkhkx1VRmYHbXX3x83saWYwOyi9af3fA0fTRT93909O93hT0d03RAporFUSEBGBDJKAu3/KzL7p7gfSRde7+7YZnPNC4M/d/fszOMa0dPUOAlBXrYXjREQggyRgZhHgejN7I1AO3GNmz7r78DTPeRFwppn9T+Ap4GPu3jHNY01JV1+QBLR6qIhIIJN7An8HXAH8E/AF4NXA52ZwzsPALcB5wH7gyzM41pSc7AnU6BkBERHI7J7AG4AL3X0IwMx+TnAF/2cTfcnMrgNuHVe8w903j/rMZ4EXpxJwS8v070kn088GrFrWTHVlOBJBLBa+LTRV53BQnWdHJkkgMpIAANw9bmZDE30h/bktwJbRZWbWYGZ/lp5yCsGm9VMaVmpr6yGZTE3lK0DwP97hYz2Ul0Xo6eqnt3tgyscoNLFYHa2t3fkOI6dU53BQnTMXiZRMePGcSRLYama3EgzbpICPAtO9MdwD/KWZPeTuj6SP9ZNpHmvKuvoGqa8u12YyIiJpmdwT+AjQBDwEPAzEgI9N52TungDeAXzNzJ4DLgD+cjrHmo6+gWFqQjIMJCKSiUx6An/t7n88Wyd09weATbN1vKnoGxiiujKTKouIhEMmPYGrsx5FjvTFh6mqUBIQERmRSYu4y8zuAR4kGNMHwN2/kLWosqQvPqyegIjIKJm0iO3pnytHlU19es4c0DcwTHWF7gmIiIzIZNmI9478bmYV7h7PbkjZkUgkGRhMqCcgIjLKaVtEM4sC3wTudveRaZw/MrNW4AMzWDYiL/riQbjVuicgInLSRDeG/w9QD/x2VNmfEEwXvTmLMWVFb3/wfJt6AiIiL5koCVxNsGLosZECdz8I3AC8NduBzbaekSSgnoCIyEkTJYFBd+8fX+juXUDB3RfoGwiSQKWSgIjISRMlgYSZvWy1onRZwU2xGRxKAhAtz+TRCBGRcJioRfw+cLuZ1YwUpH+/HfhRtgObbfHBBADRstI8RyIiMndMNDZyG/B14IiZPUOQMM4Gvkdw07igxIfSSUA9ARGRk06bBNw9CXzQzD5NsNBbEnjE3Q/nKrjZNDiknoCIyHiZPCy2F9ibg1iyaqQnUF6mnoCIyIjQtIgjPYEKDQeJiJwUmhYxPpSgBCgrDU2VRUQmlfNJ82a2iGCG0WKgD3i3u+/J9nnjgwnKyyLaVUxEZJR8XBb/M/Azd9+Y/v0fcnHSwaEE0XLdFBYRGS2nPQEzmwecD/x+uujbwH/l4tyDQ0ndFBYRGSfXreJqYB/weTN7DLgLGMzFieNDCaJKAiIiY2StJ2Bm1wG3jiveCWwEPuXuf25mNwHfAS7P9LgtLbXTimdwKEF1VTmx2MtWwihqYasvqM5hoTrPjqwlAXffAmwZXWZmq4En3P3f0kV3Al+cynHb2npIJqe+sVl8KEEEaG3tnvJ3C1UsVheq+oLqHBaqc+YikZIJL55zOj7i7i8CB8zsjemiNwOP5+Lcg0MJ3RMQERknH+sq/wHwDTP7HNAF3JiLk8aHEtRoGWkRkTFy3iq6uzOFewCzZXAoQVNNNNenFRGZ00IzPjI4lKRMw0EiImOEplVMJFOURvS0sIjIaOFJAokkpZHQVFdEJCOhaRWHEylKS9UTEBEZLTRJIJlMUqrF40RExghNEhhOqicgIjJeaJKA7gmIiLxcaFrFRDJFRLODRETGCEUSSCZTpFJQpiQgIjJGKJJAIr3gnO4JiIiMFZIkkATQcJCIyDghSQLpnoBuDIuIjBGKVvGlJKCegIjIaOFIAgklARGRUwlHEkjfE1ASEBEZK6f7CZjZfOCeUUUNQMzdp7dxcIaSmh0kInJKOU0C7n4M2ABgZhHgv4BPZvu8I/cENDtIRGSsfA4HvRfoc/c7s32iBc3VvOv1xrmrWrJ9KhGRgpKXTXfNrJSgB3BNLs4XKSnh+ivPorW1OxenExEpGCWpVCorBzaz64BbxxXvcPfNZnYV8HF3v3IKh1wB7J6t+EREQmYlsGd8YdZ6Au6+BdhymrevBf5lOsdta+s5eaN3KmKxutD1BFTncFCdw2G6dY5ESmhpOf3cm3zdE3gV8ECezi0iImn5SgKrgAN5OreIiKTl5cawu1fn47wiIjJWKJ4YFhGRU1MSEBEJMSUBEZEQUxIQEQkxJQERkRBTEhARCTElARGREFMSEBEJMSUBEZEQUxIQEQkxJQERkRBTEhARCTElARGREFMSEBEJMSUBEZEQUxIQEQmxnG8qY2YrgP8H1AOdwI3uvjfXcYiISH56ArcA33f3DcCPgE/nIQYRESE/20uWEvQCAGqA/il8j0ikZNonnsl3C5XqHA6qczhMp86jvlN6qvdLUqnUDEKaOjNbDTwEDANR4FXu/kIGX70UeCCbsYmIFLHLgAfHF2YtCZjZdcCt44p3AJXA59z9p2b2NuBm4Dx3nyyQCuAi4DCQmOVwRUSKVSmwCHgMiI9/M6c9ATOLAc+5+7xRZa3AOndvzVkgIiIC5P7G8HFgwMwuAzCzS4BuJQARkfzIxz2Bi4EvAVVAN/BRd38yp0GIiAiQhyQgIiJzh54YFhEJMSUBEZEQUxIQEQkxJQERkRDLx7IRWWVm1wN/A5QDt7n7V8a9vwG4nWDpivuBD7n7cM4DnUUZ1Pka4G+BEmA38F5378h5oLNosjqP+txVwJfdfWUu48uGDP7OBnwDaAKOAH9Y7H9nM9tEUOcosB/4I3fvzHmgs8jM6glWVbja3feMe2/W26+i6gmY2RKCBekuBTYAHzSzdeM+9l2CaalrCRrFD+Q2ytk1WZ3T/0F9DbjK3c8HthE8pV2wMqzZKBwAAAR6SURBVPw7Y2YLgH8k+DsXtAz+ziXAvwJ/n/47Pwn8VT5inS0Z/p3/Cfjf6To78Be5jXJ2mdkrCJZ2WHuaj8x6+1VUSQDYDPzK3dvdvRe4C3j7yJtmthyocveH00V3ANflPMrZNWGdCa6gPuLuB9OvtwHLchzjbJusziNuJ+gBFYPJ6rwJ6HX3f0+//gxwyt5RAcnk7zx6QcpqMl+Qcq76APAR4ND4N7LVfhXbcNBigrWFRhwGLp7k/TNyEFc2TVhnd28DfgJgZlUEV4dfymWAWTDZ3xkz+zjwBPAwxWGyOq8BjpjZt4CNwHPAx3IXXlZM+ncG/hy4x8xuA3qBV+Qotqxw95sAgpG9l8lK+1VsPYEIMPrptxIgOYX3C1FGdTKzBuDnwFPu/p0cxZYtE9bZzNYDbyPYu6JYTPZ3LgMuB77m7puAXcAXchZddkz2d64CvgVsdvdFwFcJNqwqVllpv4otCRwgWC1vxELGdqsme78QTVonM1tEsAz3NuCm3IWWNZPV+br0+78DfgEsNrNCX4Z8sjofAXa6++/Sr7/Py6+aC81kdV4P9Lv7o+nX3yBIhMUqK+1XsSWBe4HXmVnMzKoJrgZHxkhJb2M5kF64DuA9wC9zH+asmrDOZlYK/Az4obv/aQZLdheCyf7On3L3tend694EHHL3y/IU62yZsM4Es0liZnZ++vWbgcdzHONsm6zOLwBL7aWxk2sIlksuStlqv4oqCaRvfn4SuA/YCtzp7o+a2S/M7ML0x94N3GpmO4Ba4Iv5iXZ2ZFDntxDcNHy7mW1N/7s9jyHPWIZ/56IyWZ3dvR94K/BNM3sGuAL47/mLeOYyqHMH8MfAD81sG/A+4L15CzhLst1+aQE5EZEQK6qegIiITI2SgIhIiCkJiIiEmJKAiEiIKQmIiISYkoCISIgpCYiIhJiSgMgEzOxmM7u5UI4rMlXFtoqoSE6Y2a+B84GF7h7Pczgi06aegMgUmdkK4DyC5Zrfkt9oRGZGPQGRqbuBYI+GR4AbgS0AZlYLnADOcPfD6bL1wH8Ca929Oz/hipyeegIiU3cDwVLNdwGvTW9jibv3ADsIFuwb8ffAZ5QAZK5SEhCZAjO7FKgB7nP3duBXwPWjPvIY6SRgZq8B1hGsc4+ZfdjMTrd3rEheaDhIZGpuBH7g7on06+8Dfwncmn79GMHeuACfBf6Xuw8CuPtXcxmoSCbUExDJUHo7w3cQNPwjfgqsGbWZy2PAJjN7G1A1+rNFsLuZFCH1BEQydy3QDjxlZpXpsgTBFpY3EGzi8hTBtn+fB/6buycBzGwecCznEYtMQj0BkczdCKwA+sf9uw54t5mVpZ8Z2A7scffRW/+dR7DHs8icop6ASIbc/Q2TfcbMosB84KPj3lISkDlJPQGR2fUp4Lfu/vC48nNREpA5SD0BkYn9OpMPmdkmgg3RtxFs+D6Gu79/OscVyTZtNC8iEmIaDhIRCTElARGREFMSEBEJMSUBEZEQUxIQEQkxJQERkRBTEhARCTElARGREPv/7X0TpFLS6P0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cross_entropy1 = np.log(delta_y)\n",
    "plt.plot(delta_y, cross_entropy1)\n",
    "plt.ylim(-8, 1)\n",
    "plt.xlabel('$|\\Delta y_i|$')\n",
    "plt.ylabel('Cross Entropy Loss')\n",
    "# plt.savefig('CELoss_vs_delta_y.png', dpi = 300, bbox_inches = \"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lec5_utils import loss_plot_3d\n",
    "from lec5_utils import loss_contour_plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigma(x):\n",
    "    return 1 / (1 + np.exp(-x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import log_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6121919007930319"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_loss([\"win\", \"lose\", \"win\", \"lose\"], [0.9, 0.8, 0.6, 0.2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6121919007930319"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_loss([1, 0, 1, 0], [0.9, 0.8, 0.6, 0.2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mean_ce_loss(theta0, theta1, x, y_obs):\n",
    "    y_hat = sigma(theta0 + theta1 * x)\n",
    "    return log_loss(y_obs, y_hat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "nba = pd.read_csv(r\"data_set\\nba.csv\")\n",
    "nba[\"WON\"] = nba[\"WL\"]\n",
    "nba[\"WON\"] = nba[\"WON\"].replace(\"W\", 1)\n",
    "nba[\"WON\"] = nba[\"WON\"].replace(\"L\", 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5725397505600823"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean_ce_loss(theta0=-11, theta1=0.1, x=nba[\"PTS\"], y_obs=nba[\"WON\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "theta0s = np.linspace(-11, 0, 10)\n",
    "theta1s = np.linspace(0, 0.3, 10)\n",
    "loss_plot_3d(theta0s, theta1s, mean_ce_loss, nba[\"PTS\"], nba[\"WON\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "theta0s = np.linspace(-11, 0, 20)\n",
    "theta1s = np.linspace(0, 0.3, 20)\n",
    "loss_contour_plot(theta0s,\n",
    "                  theta1s,\n",
    "                  mean_ce_loss,\n",
    "                  nba[\"PTS\"],\n",
    "                  nba[\"WON\"],\n",
    "                  flip_axes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_train_and_validation_ce_loss(C, degree):\n",
    "    c_model = Pipeline([('scale', StandardScaler()),\n",
    "                        ('poly', PolynomialFeatures(degree=degree)),\n",
    "                        ('model',\n",
    "                         LogisticRegression(fit_intercept=False,\n",
    "                                            C=C,\n",
    "                                            max_iter=10000,\n",
    "                                            solver='lbfgs'))])\n",
    "    c_model.fit(p3_train[[\"fare\", \"age\", \"pclass\", \"sex\"]],\n",
    "                p3_train[\"survived\"])\n",
    "\n",
    "    training_predictions = c_model.predict_proba(\n",
    "        p3_train[[\"fare\", \"age\", \"pclass\", \"sex\"]])\n",
    "    train_ce = log_loss(p3_train[\"survived\"], training_predictions)\n",
    "\n",
    "    validation_predictions = c_model.predict_proba(\n",
    "        p3_validation[[\"fare\", \"age\", \"pclass\", \"sex\"]])\n",
    "    validation_ce = log_loss(p3_validation[\"survived\"], validation_predictions)\n",
    "\n",
    "    return train_ce, validation_ce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEQCAYAAACugzM1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXicZbn48e87S7amSdMmbdJ0L81duq9QlgKyW9yQRUVwRfRSQX8HPbIKiBUO4lGOAiqLeISiCAdBRJTaAi0FSikF2tKb7lvSJk26ZE9m+f0xkzotyWTSZvLOTO/PdeXKvOvcT9Pknmd5n8cJh8MYY4wxXfG4HYAxxpjUZonCGGNMXJYojDHGxGWJwhhjTFyWKIwxxsTlczuAXpYNzAaqgKDLsRhjTLrwAmXAm0Dr4QczLVHMBpa4HYQxxqSpucDSw3dmWqKoAti7t5FQKHWfDxk0KJ/a2ga3w+gVmVKWTCkHWFlSVSqXxeNxKCrqB9G/oYdLaqIQkcuAmwA/8AtVvTfm2DTgkZjTS4C9qjpJREYAjwKDAQU+r6qJ/AsHAUKhcEonCiDl4+uJTClLppQDrCypKg3K0mmTfdI6s0WkHJgPnApMA64SkQkdx1V1lapOU9VpwMnAXuAb0cP3Afep6nhgBXBzsuI0xhgTXzJHPZ0NLFLVOlVtBJ4ELu7i3OuBl1V1qYj4gdOi50Ok1nFJEuM0xhgTRzKbnoZyaHtXFXDC4SeJSCFwFTA5uqsYOKCqgZjrhvXkjQcNyu9xsH2tpKS/2yH0mkwpS6aUAzKjLKFQiOrqaurq9hIMpv8gxupqtyMAr9fLwIFFFBcX4/EkXk9IZqLwALENcg4Q6uS8y4G/qGrHP+Ph19HFdV2qrW1I6bbAkpL+1NTUux1Gr8iUsmRKOSBzylJXV43P56GwsASv14fjOG6HdFR8Pg+BQI/+lPWqcDhMMBhg37597N1bz8CBgw8e83icuB+wk9n0tIPIuNwOpUBlJ+d9CvhjzHY1UCgi3uh2WRfXGWMyWFtbC0VFxfh8/rRPEqnAcRx8Pj8DBgyira2lR9cms0axELhVREqARuAiIk1MB4mIA8wEXuvYp6rtIrIE+AywAPgC8PckxnlQ5XqleX9d5wed2JexG86/9zoc+tpxcKJfsa8PFORS39CKx/GAx4PH4+B4vXgcDx6vF48n8t3r9eL1+/D5fHi8PhyvDzw+8HjtF8ccA8I4joceNiiYbkT+TXvW4pK0RKGqO0XkRmAxkAU8qKrLReR54IequoLIkNg2VT08vX0T+L2I3ARsAz6XrDg7NDc20G/RnfR3+qbJqiCBc0LRr/ZOjgXDHgKOlyA+gk7kK+TJIuTNIuzLxvHn4mTn4svph79fATkFheQWFOHtNwAntxAnrxDH4+3kzsYYcygnwxYuGgVsPtI+iuptW2javzeyEfPvcsidYv+9oq/DYQgThnAYJ3p+5N81si8U2RHZFw6Rl5dFY0Mr4XCIcDhMOBQkHIq+DocIB0MQDhIKhgiHApFjwWDkdTAAwY7v7RBqxwm244Ta8Yba8Iba8YXbyXHayHXayXXa8Dkf/kQWwqHNX0i43yCyi4aQM3gY3qKheAYOx+k3MOEaS6a0h2dKOSBzyrJr11aGDRvtart+rIaGBubPv5U77rg7ofPXrVvLX/7yFNddFxnd73YfRaxdu7ZSWjry4HZMH8VoYMvh52fak9lHZfCIUURyTXIl+xc5FA7T1BKgsbmdPY2tHDjQQNP+fTQd2Evb/jqCjftwmvfSv7WegU31lNTtxLN52cHrg1n98Q8Zi690LL7yiXiKR+H0YISEMZmovv4A69drwuePHz+B666b0P2JacASRQbyOA75uX7yc/0MGZgHFAHDP3RefVMbVbVNrKtuoLJyN427t5NdX8mI1j2MbN7MkO2raHvzKcJZ/fAPn4Rv7An4hk/B8fr7vEzGuO0Xv/gpe/bUcP3132Pr1s0UFg4gOzub+fPv4o47bqemppo9e2qYNesErrvuZt5++y0efvi3/OpXv+Xb376KiRMnsWrV2+zbt5fvfvf7nHTSKW4XKWGWKI5h/fOy6J+XRcXwATBzGDCTxpZ2Pti2j+Vb97Jh4w4GNW1C/FVM2fQuuRvfgKw8/GNm4590Nt6BH04+xiTLq+9VsfTdTqciOmqnTinjlMllcc/57ne/z9VXf51rrvkPLrnkE/z5z7+krGwoL774AuPGVfDjH/8X7e3tXH75Jaiu+9D1gUA7v/nN71i69BUeeOB+SxQmffXL8TO9ooTpFSWEzx7HjpoTeEur+fmaSooaNzMnbyuTdRnt617GO2wSTad9mnC/0TYKyxxTiooGUlY2FIBzzjmftWtX88QTC9iyZTP79++nubnpQ9fMmXMyAGPGjKW+/kCfxnu0LFGYLjmOw/DB+QwfnM8nTh3N2i3H89Lblfx5ww7m5q7nzCpl14If4R02ieyTPoe3qNztkE0GO2Vy95/6+0p2dvbB108++UdeemkRn/jEhVx88Qls3ryRzgYJZWVFrnEcp9PjqcwShUmIx3GYNHoQk0YPYlfdWJ59dQQ3rDmeM/pt4KNV7xB88mb8E88ie9ancbJy3Q7XmF7n9Xo7nUrkzTff4BOf+DTnnns+69atZf36DwiFQj2aIiPVWaIwPVY6MI+rPj6RC04axdNLy/ihjuSzxWuZvHohge3vknvWN/EWj+z+RsakkYEDBzFkSCk/+clth+y/9NLLuPvuO3j00d/Rr18+kyZNoaqqkvLyHk1Rl9LsOQoXZMo4d4Di4nyee2UDjy9cT2n7Dr428DWyg01kz/kM/olnp03fRSb9TDKlLKn2HMXRSufnKDKnbmRc4TgOcyaUMv9rc8gfNZHbqs9np38krcseo3XJI4RDqfGLYYw5cpYoTK/Iz/XzrU9P5ry5E/hp1cm85sykfd3LtCz6deQpcmNM2rI+CtNrPI7Dx04exajS/tz/jJ/6HD/nbnqd5vZmcs/5No4vu/ubGGNSjtUoTK+bNGYQ3//cNF5qncCzwbkEtq+m5V+/tmYoY9KUJQqTFKNKC/j+56bzRlsFzwfmENj6Nq3Ln3A7LGPMEbBEYZJmxJD+/Odl01nWPoE3mUT7uy/Qtnax22EZY3rIEoVJqmEl+Xz705P5077pbPONpPXVPxDYudbtsIwxPWCJwiRdxfABXH7e8fyq+iTqfUW0vPQA4dZGt8MyJmnmz7+V55//K3v21PC9713T6Tmnnjor7j0qK3dyxx0/AiJrW9x55+29HmeiLFGYPjF36lBOnz2W3+w5kVDjflpefdTtkIxJuuLiEu6++3+O6Npdu6rYuXMH0LG2xc29GVqP2PBY02cu/chx3L27noV1Uzhnw2u0j56Jf3T8T1XGdGj/4FXa9ZWk3Nsvp+GviD/t9w03fJ9zzz2fM844C4CvfOVyrr76//Hb395Ha2sL9fUNXHPN/2Pu3DMOXlNVVcnVV3+dJ5/8K5WVldxyy400NzczceKkg+fU1FRzxx2309BQz549Ncyb93GuvPIb3HPP3VRW7uRnP/svPvKRsw6ubbFt21buums+9fUHyMnJ5bvf/R7HHz+R+fNvpV+/fFTfZ8+eGr70pSu54IJP9Mq/j9UoTJ/xeBy+csHxLGqbSo2nhJYljxBqTq/pls2x67zz5rFw4T8A2L59G21tbTz11J+47rqbefjhx7juupt44IH7u7z+Zz+7k3nzPs4jjyxg8uSpB/e/+OI/OOec8/jtbx/hf//3TzzxxOPs27eP73zne4gcz7XX/uCQ+9x++81ccsln+f3v/8jVV/8HN930A9ra2gCort7Nffc9yJ13/jf33ntPr5XdahSmTxUX5nLpWcKD/9zLDwY8T+trj5N75tfdDsukAX/FKd1+6k+mk08+lZ///C6amhpZuPAfnHfeR7n00stYtmwJixcvZM2a92hubu7y+pUr3+KWW+YDcO65Hz3Y53DZZVewcuUKFiz4A5s3byQQaKelpfP7NDU1sWPHDk4//UwAJk2aTEFBAdu2bQXghBNOxHEcxowZy4ED+3ut7FajMH1u7pQyBo8ay+KWCQQ2vEawZrPbIRnTLb/fzymnzGXp0ldYtOhFzjnnfL71ra/x/vtrEBnPF77wlbjrTDiOc3CyUsdx8Hi8APzylz/nz3/+I6WlZXzxi1+lsHBAl/cJhz/80Go4zMHpz2PXvOhNlihMn3Mchy+eP54lgSk0O7m0vv6ntFvIxRybzjtvHn/846MUFg4gLy+P7du38tWvfoM5c05hyZKXCcWZfWD27BP5xz+eB+DllxfR1tYKwIoVb3DZZVdw5plns23bVmpqqgmFQni9vg+tf9GvXz5Dh5bz8suLAFi9+j3q6moZM2ZskkocYU1PxhVF/bM5f67w3JLJXFK1nOC2VfhGTnc7LGPimjJlGg0NDXzqUxdTUFDIxz72Sa644lJ8Ph8zZsympaWly+ana6/9AbfeehPPPvs048cfT15ePwAuv/xL3H77D8nOzmbw4FLGj59AZeVOKiqEhoZ6br/9Zi644JMH7/PDH97OT3/6Ex566Df4/VnMn38Xfr8/qeVO6noUInIZcBPgB36hqvcedlyA3wBFwC7gs6q6V0S+CNwJ7I6e+jdVvTGBtxyFrUfRp46mLIFgiFseeo2v8WeKC3Pod8mPcTzufHaxn0nqsfUokidl1qMQkXJgPnAqMA24SkQmxBx3gGeBO1V1KvA2cF308CzgP1R1WvQrkSRh0ozP6+HiMyr4v/rphPfvon1dcoY+GmOOTjL7KM4GFqlqnao2Ak8CF8ccnwE0quoL0e2fAB01jtnAF0XkPRF5VESKkhincdG0ccUESiexJTiE1refIxyytSuMSTXJTBRDgaqY7SogdhHZ44BdIvKQiKwE7gcaYs69HZgCbAd+lcQ4jYscx+EzZ1XwYtMEaKwjsOlNt0MyKcPpdJSPOTqRf9OejYpKZoOwB4jtKHCA2J+6DzgDOE1VV4jI7cB/A19S1Qs7ThKRu4CNPXnjaFtbSisp6e92CL3maMtSUtKfxRNPpHrLSoau/gdlc85xZa1t+5mklqamfOrq9lBYWITX60ub9dfj8fncG2gaDocJBgMcOLCXgoL8Hv0fSWai2AHMjdkuBSpjtncB61V1RXT7ceBJESkEvqKqP4/ud4AetUdYZ3bf6a2ynDVjGH9/fwKfq36NXe8sx1c+ofuLepH9TFJPbm4RubktVFdXEQoFu78gxXk8nrjDZ/smBi+5ufnk5hYe8n8kpjO7U8lMFAuBW0WkBGgELgKuijm+DCgRkamq+g7wceAtIs1P/ykiy1T1DeDbwNNJjNOkgOGD82ktn0X93lU4q54nv48ThUk9juNQUjIYx8l1O5Rekc4JPKF6kIj0j36fLiJXiEi3g3ZVdSdwI7AYWAUsUNXlIvK8iMxS1WbgQuABEVkDnAlcq6pB4FLgfhF5H5gJ/OeRFM6kl/NOGsvLzUJ452qCtdvdDscYE9VtjUJEfgQcJyLXAS8Aa4DTgSu7u1ZVFwALDts3L+b1G8AJnVy3hMioKHMMqRg+gL8Omk1b02q8775A3ke+5nZIxhgSq1HMI5IULgIeV9UzganxLzHmyJx1kvBm6xjaNy4n3NbkdjjGGBJselLVJqLPRUR3ZSctInNMmzp2EBvzJuEJtdO+4Q23wzHGkFiiqBWR+4g8Lb1QRO7k0NFLxvQax3GYMGMGVYEBNKx+ye1wjDEklii+QCQxXBCtWYSBLyY1KnNMmzOxlBWBcfj3bSW4d6fb4RhzzOs2UajqbuCe6ENx04G1QF3SIzPHrNxsH86YOQTDDs1rbP4nY9zWbaKIjnr6jYiMIDLq6ctEptswJmlOmjWONe3DaPtgqc3/ZIzLbNSTSUmjSgvYlDcZf6CRwNZ33A7HmGOajXoyKWvE9DnsD+Wy792X3A7FmGOajXoyKevEiUN5NzAG/+61hNu6XrTeGJNcPRn1NM9GPZm+lJPlIzBsOl6CNG9a6XY4xhyzEh319AdgkohcCTwU3WdM0sn06ewL5bJ3zTK3QzHmmJXIqKfzgBXAp4BPAG+KyCfjX2VM7xg3YiAaHkNe7TrC7S1uh2PMMSmRpqfbgdNV9UJV/QSRNbBvTWpUxkR5HIfQiBn4CNKwwZqfjHFDIokiS1XXdmyo6hrAm7yQjDmUzJjFgVCONT8Z45JEEkWziMzq2Ii+tmk9TZ8ZUVrABs8Y8uvWEQ60uh2OMcecRFa4+0/gORFZT2TE03jgkqRGZUwMx3FwRs7Cv3Ut+3QlRRNPcjskY44piYx6WgJMAH4C3AUcr6ovJzswY2LJzNk0hLKt+ckYFyS0Zraq1gF/79gWkaWqemrSojLmMEOK+/Oabwzj9inhUADHk8zl3o0xsRKawqMTU3o1CmMS4BkxlWzaqNuwxu1QjDmmHGmiMKbPjZp6AoGwhz1r33Q7FGOOKZYoTNoYMmQgO50ysmvedzsUY44pXTb0isj/dHHIAbKSE44x8bWWTGBkzYvU11TRv6TM7XCMOSbEq1HUdvG1B7gj+aEZ82GDJ80GYPuqN1yOxJhjR5c1ClW9rS8DMSYR5WOPY8eifMI73iMy/ZgxJtmSOsZQRC4DbgL8wC9U9d7DjgvwG6AI2AV8VlX3RpddfRQYDCjweVVtSGasJj14PB72FVRQfuBdWppbyMnNcTskYzJe0jqzRaQcmE9kEsFpwFUiMiHmuAM8C9ypqlOBt4HroofvA+5T1fFEZq69OVlxmvTT/7gZZDkBNq9a4XYoxhwTEplm/EhrHWcDi1S1TlUbgSeBi2OOzwAaVfWF6PZPgHtFxA+cFj0f4BFsyhATY/iUmbSHvTRuetvtUIw5JiRSo9gqIvNFZGQP7z0UqIrZrgKGxWwfB+wSkYdEZCVwP9AAFAMHVDXQxXXmGOfLzmVPzggG1q8nGAq5HY4xGS+R2sIc4ErgFRF5j8gf9OdVNdzNdR4ikwh2cIDY32ofcAZwmqquEJHbgf8GbjzsOg67rluDBuX35HRXlJT0dzuEXuNGWXaOnUbB2qepq97NhMkVvXJP+5mkJiuL+7pNFKq6HbhFRG4DPg78kkgT0b3A/6hqV/M+7wDmxmyXEll7u8MuYL2qdjQ0P06kuakaKBQRr6oGgbLDrutWbW0DoVB3ecw9JSX9qampdzuMXuFWWQaMnUJo7dOsf2MZJaVH/zyF/UxSk5Wlb3g8TtwP2Al1ZovI8cBPgV8Dq4CrgVHAE3EuWwicJSIlIpIHXAS8EHN8GVAiIlOj2x8H3lLVdmAJ8Jno/i8QMyGhMQB5pSNpdnLxVq9zOxRjMl4indlLgUVAMzBbVT+tqn8lkizmdnWdqu4k0oy0mEhyWaCqy0XkeRGZparNwIXAAyKyBjgTuDZ6+TeJjJJaG32Pm464hCYjOY6HhsKxlAd3ULe/2e1wjMloifRR3Af8OfpJ/yBVDUWfd+iSqi4AFhy2b17M6zeAEzq5biuR/gtjupQ/egq5+1azZu065pw03e1wjMlYiTQ9PQF8TUSeEZH/E5Evdxywh+CMmwZVRFot6ze+63IkxmS2RBLFPcClwDPA88BXROTHSY3KmAR4C4fQ6Cskf/9G2gM2TNaYZEkkUZwLnKOqD6vqg8A52ANwJkUES4Qx3l18sK3O7VCMyViJJIoaDu3LCAH7khOOMT1TVDGNPE8bW9faqnfGJEsindmrgCUi8ggQIDJsdY+I/AeAqv538sIzJr6c4RNpBAI71wCnux2OMRkpkUSRC7wHzIxub4l+n8yHn6A2pk958gppyhnC0PrtVO9tYnBRntshGZNxEnky+8sA0bme/Kq6IelRGdMD/mETGL3+Fd7dWM3gWaPcDseYjJPIA3fHRR+IWwW8JSIbRWR88kMzJjH9R08iywlSvX6t26EYk5ES6cz+FXCXqhapaiHwYyIP4RmTEnxlAoBnz3oCQRsma0xvSyRRDFHV33dsqOrvgJLkhWRMzzg5+bT2K2WUZxcbd+53OxxjMk4iicInIgM7NkSkGOvENikmZ9jxjPbVsGZTjduhGJNxEhn19EvgdRH5E5EE8Vng50mNypgeyh5+PGFdzJ6NCmf0zvoUxpiIRBLF74D1wPmAF/imqi5MalTG9JC3NNJPkVe/mQONbRT0y3I5ImMyRyKJ4k1VnUZkunBjUpInr5BA/hCOa9vNmi11nDSx1O2QjMkYifRRNIqIrVltUl7OsOMZ669mzcY9bodiTEZJpEbRD9gsItuBg9OKq+qUpEVlzBHwDRVy1r1E7bYNhMIT8TiO2yEZkxESSRTfSXoUxvQCb1nkOdCy9h3sqG5gxJD0XMjemFSTSKL4gqp+NXaHiDwJvJyckIw5Mp5+RYTzSxgb7aewRGFM7+gyUYjI/UA5MFdEYh+w8wNjkh2YMUciq3w84xqW87+bavnoiSPdDseYjBCvRvEQMAmYCjwVsz8AvJ7MoIw5Ut4yIVeXcKByK63tU8n2e90OyZi012WiUNUVwAoRWaiqO/owJmOOmLc08rDdSM8u1m/fx6Qxg1yOyJj0l0gfxXAR+QMwEDg4jMRGPZlU5PQvgbwBHNdWzerNdZYojOkFiSSK3wCPACuxOZ5MinMcB19pBRXNa1m4uRYY53ZIxqS9RBJFwJY7NenEW1ZB/qblNO+tZm99K0X9s90OyZi0lkiiWC0ik1X1vZ7eXEQuA24iMlLqF6p672HHbwG+AuyN7npAVe/tan9P398cmzr6Kcb4drN2Sx2nTC5zOSJj0lsiiWIMkZXttgLNHTu766MQkXJgPpG1tluBZSKyWFVjlyGbBXxWVV877PKu9hvTLU/RMMjKZXxuLas3W6Iw5mglkihuPMJ7nw0sUtU6OPiQ3sXAj2LOmQXcEF2P+xXge6raEme/Md1yPB68Q8ZRUbWTv2yuIxQO23QexhyFLicFFJERAKr68uFfQG4C9x4KVMVsVwEHJxcUkXzgbeD7wAxgAHBzV/t7VCpzzPOWVVAYqCXcUs+23fVuh2NMWotXo/gLkT/UiMhTqnpRzLGfAC90c28Ph46ScoCDCxqragMwr2NbRH4GPKyqN3a2nx7UbAYNyk/0VNeUlGTO9BKpWJaW8dOpXP4kY3zVbN7dyOzJ5d1ek4rlOFJWltSUrmWJlyhi6+qHT9mRSD1+BzA3ZrsUqOzYiNZYzlbVh2Pu2d7V/gTe76Da2gZCodQdyVtS0p+amsz4lJuqZQn7B4PXx7TCvbzxXiUfmRq/nyJVy3EkrCypKZXL4vE4cT9gx0sU4S5ed7bdmYXArdF5ohqBi4CrYo43A3eJyGJgC/At4Ok4+41JmOP14y0Zw7h9NSyoPEBTS4C8nES65Iwxh4u3cNFR9f6p6k4izUWLgVXAAlVdLiLPi8gsVa0Bvg78FdDo+/2sq/1HE4s5NnlLKyho3YU31Mb7W/d2f4ExplPxPmJ5RKSIyB9qb8xriKyd3S1VXQAsOGzfvJjXT3HohINx9xvTE96yCpxVzzEut47Vm2uZecgkyMaYRMVLFJOBPfw7OdTGHEvdDgBjorxDxoHjcMLA/Ty7qY5wOIxjw2SN6bF4s8cmsp62MSnLycrFM3AEY9qqqT3Qwq66JsoG9XM7LGPSjiUDk9G8ZRX0b9qBhxCrN9W5HY4xackShclo3tIKnGAb0wY28t7m2u4vMMZ8iCUKk9E6JgicNXA/um0fbe1BlyMyJv30KFGISFbH1B7GpANPXiFO4RBGe3bTHgih2/e5HZIxaafbRCEiF4rIL0WkP/AB8I6IfCf5oRnTO3ylFeTVbyHb7/DuBmt+MqanEqlRXA/8lsiT1a8BI4ArkhmUMb3JW1oBrY2cVB7mnY17CIdtdLcxPZFIonCiixadDfxdVesTvM6YlOAtEwBmFO1jz/4WKvc0uhyRMeklkT/4IRG5FDgP+KeIzCNmFlhjUp3TvwQnbwDDo7Pev7PRmp+M6YlEEsW1RCbzu0FVdxGZv+mapEZlTC9yHAdvaQXePRsZMbgf72zY43ZIxqSVbqfTVNWlRJqdEJEs4HOqui3ZgRnTm7ylFQQ2LefEMX6efGsfDc3t5Of63Q7LmLRgo57MMaGjn2JKQR3hMLy3yZqfjEnUkYx6GomNejJpxjOwHLL7MbBpKwV5fmt+MqYHjmTU04EErzMmZTiOB19pBcGqdUwZW8zqTXUEQzYmw5hE9GTU0/nYqCeTxrxDxxOur2HGMC9NrQE27NjvdkjGpIWejHq63kY9mXTmLRsPwLisanxeD29pjcsRGZMeuk0UqrpUVc8mUps4TlVPUdVlfRCbMb3KM3A4ZOXirfmASaMH8tYHNfaUtjEJSGTU0zgRWUNk3eu3RGSjiIxPfmjG9C7H48FbKgSqlJlSwt76VjZX1bsdljEpL5Gmp18Cd6lqkaoWAj8G7ktuWMYkh2+oEN6/m6nlPrweh7e02u2QjEl5iSSKIar6+44NVf0dYKvUm7TU0U+RXbeR8SOLeEut+cmY7iSSKHwiMrBjQ0SKAfvNMmnJM2gE+HMJVq1jppRQva+ZHTU2SaAx8STa9PS6iNwuIj8ClgH3JzcsY5LD8Xjxlo4jWKXMGFeC42DNT8Z0I5FE8Tvg60AWkAd8U1UtUZi05S0bT2hfFflOMxXDBtgwWWO60e2kgMCbqjoNWNzTm4vIZcBNgB/4haree9jxW4CvAHujux5Q1XtFZBrwIFAAvAJ8Q1UDPX1/YzrjGzqeNog2Pw1lwcL17KiuJ9txOzJjUlMiNYpGERnW0xuLSDkwHzgVmAZcJSITDjttFvBZVZ0W/epIJI8C31bVCsABvtbT9zemK57ikZF+ip1rmVFRggMsfafS7bCMSVmJ1Cj6AZtFZDvQ0LFTVad0c93ZwCJVrQMQkSeBi4EfxZwzC7hBREYSqTl8DxgC5Krq69FzHgFuw/pFTC9xPF58Q8cT2LmWgaflICMGsHjFds6cWobjWLXCmMMlkiiOdErxoRBdUiyiCjihY0NE8oG3ge8DG4gkhJuB5zq5rsc1GmPi8ZZPJLD1bUIHqjlpYim/+/s6NlUdYOzQQrdDMyblxE0UIjIIeF81MixERM4E3lNNqPfPw6HDaAQ8L7IAABb0SURBVB1iJhNU1QZgXsx7/Qx4GHg+3nWJGDQovyenu6KkpL/bIfSadCxL2+TZ7Fj2KLkHNnH+qWfw2IsfsGpjHXOmZsZnknT8mXTFyuK+LhOFiEwEXiIyIeDT0d2fBh4TkTNUVbu59w5gbsx2KXCwIVhERgBnq+rD0V0O0B69rqyr6xJRW9tAKJS6j3qUlPSnpiYzpo5I17KEwwU4eQPYt24lucPmcOKkMl5euYNPnjwSnze9Z9FP159JZ6wsfcPjceJ+wI73G3En8B1V7UgSqOq3iSxkdFcC770QOEtESkQkj8jCRy/EHG8G7hKR0SLiAN8CnlbVrUCLiJwSPe8K4O8JvJ8xCXMcB2/5RIKV7xMOhzhj5jAamttZvanO7dCMSTnxEsUIVV1w+E5VfQQY292NVXUnkSnJFxOZUHCBqi4XkedFZFa0+errwF8BJVKj+Fn08s8DPxeRdUA+8D+JF8mYxPjKJxBuqSdUu50ZMpj+eX6WrdnldljGpJx4fRTBOMfaErl5NNEsOGzfvJjXTwFPdXLdO8R0fBuTDN7yyGjtYOVafMdP4oTjh/DyqkqaWtrJy/G7HJ0xqSNejWJ39MG3Q4jIdMAmxzFpz9OvCM+AMgI71wJw8qRSAsEQb66zKT2MiRWvRnE78IyI3EZkficPcBLwQ+DKPojNmKTzlk+gXZcQDrYzqrQ/5SX9eOntSk6bOtSeqTAmqssaRXQVuyuAy4HlRJLFxcDnVfXFvgnPmOTylk+EQBstOz7AcRzOnF7O1t31bKo64HZoxqSMuM9RqOorwJl9FIsxfc43dDw4Hpo3rYJJI5gzsZQnXtrI4pU77eE7Y6LSe8C4MUfJycrDWzqOpg0rAcjN9nHypFKWv19NfVNCYzaMyXiWKMwxzzt8Km3VWwg1RiYxPnN6OYFgiKXvVXVzpTHHBksU5pjnGxGZ3zKw/V0AykvykeEDWLxyJyFbJtWYhCYFJDq760AiD8UBoKorkxWUMX3JU1SOt6CY4LZ3YfzpAHxkRjm/fmYNqzfVMmVsscsRGuOubhNFdPnT7wHV/HuyvjAwJolxGdNnHMch77gZ1L/3CuFgO47Xz4yKEgrzs/jH8u2WKMwxL5EaxRXAcapqK7uYjJU3dgb1K/9JcNd6fOUT8Hk9nH/CCP60aAMbdu7nuHIbAWWOXYn0UWy3JGEyXe6oyeDxEdj2zsF9Z0wrJz/Xz3PLtrgXmDEpIJEaxb9E5C7gGSIzvgLWR2EyiycrB+/Q8QS3vwsnfQ6A7Cwv58weztOvbGLb7npGDEnPtQSMOVqJ1Ci+BFxCZB3rjkn8nkxiTMa4wjdiKqF9VYQO/Huup7NmlJOb7eW517a6GJkx7uq2RqGqo/siEGPc5hs+hVYeI7DtHbImnQNAXo6fs2YO42/LtlK5p5Ghxf1cjtKYvpfIqKdiIh3a+USGx3qJdG5/PsmxGdOnPIVD8BQNJbD5rYOJAuCcWcP555vbefbVzXzjk5NcjNAYdyTS9PQEcDbwVWA48EV6uIa1MenCN3o2wV1KqGn/wX3987I4d/YIlr9fzcad++NcbUxmSiRRjFTVC4DngV8BpwDjkxqVMS7xjZ4F4TCBLYeO1Zg3ZwSF+Vn88V/rCdvT2uYYk0ii6Fgbcj0wKbrEqS3/ZTKSZ+AwnMIhBDavOGR/TpaPT582ho2VB3jj/d0uRWeMOxJJFNUi8n3gTeArIvJxIC+5YRnjDsdx8I+eTbDyfcItDYccO2VyGSOG5PPkSxtpa4+3UrAxmSWRRPF1oFVVlwIrgB8BP0hqVMa4KNL8FCKw9e1D9nsch8+dNY66A628sHybS9EZ0/e6TRSqWg08ICKTgeuBU1T16aRHZoxLPMUjcfoX077pzQ8dkxFFzBo/mOeWbWHnHls63hwbuk0UIjIH2Aj8DRgKbBeRk5MdmDFucRwH3+hZBHeuIdz64WRw+TkV5GT5eOi5tQRDNgDQZL5Emp5+SmR4bK2q7iDyTMU9SY3KGJf5x8yGUJDA1lUfOlbQL4srzhO27KrnhTesCcpkvkQSRZ6qru3YUNXnSXAdC2PSladkDE7+INo3vt7p8dnjBzNr/GD+smQzO2oaOj3HmEyRSKJoF5EiomtRiIgkenMRuUxE1orIehH5VpzzLhCRzTHbp4tIrYisin79LtH3NKY3OI6Dv+IUgjtWH1wi9XCXn1tBXo6P3z67llYbBWUyWCKJ4sfAy8AwEXkcWBbdF5eIlAPzgVOBacBVIjKhk/OGAHcTs3oeMAu4W1WnRb++nECcxvQqf8WpEA7Tvv7VTo8X5GVx5ccmsLOmgd//fZ09iGcyViKjnp4DPg3cArwKnKqqTyVw77OBRapap6qNRGacvbiT8x4Ebjts32zgXBF5V0SeFZHhCbyfMb3KUzAYb2kFAV3aZRKYPGYQF542htfX7ubFFTv6OEJj+kaXiUJEBnZ8AXXAn4AFwO7ovu4MBapitquAYYe9xzXASuDwhuB9wC9VdQqRqUP+mMD7GdPr/BWnEtq/i1D1xi7PueCkkcyoKOGJRRtYt7XzZipj0lm8Tuk9/HuNbIg0DYVjvnu7ubenk+sPjiUUkUnARcBZHJZAVPUbMa9/LSJ3ikihqiY0I9ugQfmJnOaqkpLMWQQnU8rSWTlCBWey9bXH8G5bTsmk6V1e+4Mvzubae17h/mdWc8e3TmVkaUEyQ+1WpvxMwMqSCuIliv8FTiayst3vYkc+JWgHMDdmuxSIXVL1EqCMyNPeWcBQEVkCnE7kwb47VTW2hzCQ6BvX1jYQCqVue3FJSX9qaurdDqNXZEpZ4pXDO2om9WuWEJ5+MY4vq8t7XP3pydzx6FvceP+rXH/5TAYPyE1WuHFlys8ErCx9xeNx4n7A7rLpSVW/RKQT+h3gHhF5TUS+KSIDEnzvhcBZIlIiInlEag8vxNz/FlWtUNVpwDygUlXnqmoIuDB6PiLyBeCNaD+HMX3OX3EqtDV/aEbZw5UMyOXaz04nGAxz9+Nvs7e+tY8iNCa54nZmq2qTqj6qqucQqQEUAotF5E/d3Tg6y+yNwGJgFbBAVZeLyPMiMquby78IfFdE1gBfBq5MoCzGJIV36PjIlB7vL+723PLifvy/S6fS0NzOTx9/m9r9LX0QoTHJ1ZMH50qiX8VAdTfnAqCqC4h0gMfum9fJeVuAUTHba4g0exnjOsfxkDXhLFrf+BPBPVvxFo+Me/7osgK+e8lU7nnyXeb/YQX/cek0hg1O/X4zY7oSt0YhIsNF5HoRWUukz2IXcKKqntcn0RmTIvzjTwNfNm3v/TOh8yuGD+D6z88A4I7HVqLbbDSUSV/xhscuBt4ABgOfV9XJqnqXqlZ2dY0xmcrJ7odf5hLY+Dqhpn0JXTNscD43XDGTAflZ3P3HVfzrrR32UJ5JS/FqFKcDA4j0D7wsIgeiX/UicqBvwjMmdWRNOgdCIdrX/Cvha4oLc7n+8plMHD2Qx178gF8/s4bm1oQH8BmTEuL1UYzusyiMSQOewiH4Rk6j/f2XyJr+8bhDZWPl5/q55uIpvPDGNv7v5U1s3V3Plz86HhlRlOSIjekdXSYKVd3al4EYkw78k88lsPVt2je8Rtb40xO+zuM4zJszkrFDC3job+/zXwve5vRpQ7nkjLHk5dgS9Ca1JTIpoDEmyls2Hs+gkbSt+hvhUM+bkGREEbd/9UTOO2E4r7xTyY0PvMFLq3baAkgmpVmiMKYHHMche9aFhA9U065Lj+ge2VlePnPmOG76wiyKB+Twvy8oNz24nDfXVROyzm6TgixRGNND3hFT8Qw5jraVzxAOtB3xfUaXFXDD5TO5+qLJ+DwO9/9lNTf85nVeXLHdOrxNSrFEYUwPOY5D9uyLCDfupX3toqO+1/RxJdz2lRP4xicnUtAvi8cXrufae1/l4b+9z5otdSk9b5k5NtiSpsYcAd/Q4/EOm0Tb28/hH386TtbRTQDo8TiccPwQTjh+CJsqD7D47R2s0GqWvldFQb8spowZxKQxA5kwaiD5udb5bfqWJQpjjlD27Itoevo22t77B9kzP9Vr9x0ztIAxQydwxblB3t1Yy/J11az8oIal71XhAEOL+zG6rIDRQwsYXpJP6aA8Sx4mqSxRGHOEvCWj8Y2ZTduq5/GPOxlPweBevX+W38us8YOZNX4wwVCIzVX1rN1cx6aqA6zasIel7/17XbCCPD8lA3Ip6p9N+ZACsrzQPzeL/Dw/+Tl+crK95GR5ycnykeXz4Pd5cBwnzrsb82+WKIw5CtknXUZg+3u0LPk9ufO+l7Q/vl6Ph+PKCzmuvBCAcDjMnv0tVO5ppKq2iV11jezZ38L2mkbe3VRHW3uwmzuCz+vB53XweT14PQ4ej3Pwu+M4eJxIH4rjdCxoH/PaASd2mfvOX3Jk/xz/vsjv99DenhlDh/uiLGfPGsZJE0t7/b6WKIw5Cp5+RWSfcAmtr/6BwPpl+CtO6ZP3dRyHkgG5lAzIZepxhx4rLs5n+8591De3U9/URnNrgJbWYOR7e5D2QIi26PdgKEwgGCIQDBEKQTAUJhQOEw6HCYUjCYkwB4ftdozejZ2zqquu9iMZ6Rs+7G5ZWT7a2jJjBFhflCXb393Co0fGEoUxR8k/4SO0b3iN1tcexzt8Mp5cd5dBdRyH3Gwfudk+11bZ6y2pvCpcT6VzWWx4rDFHyXE85Mz9MuH2ZlqXLbAZYk3GsURhTC/wDiwna8YnCWx8nfb3X3I7HGN6lSUKY3pJ1vSP4R0+mdZljxGs3uR2OMb0GksUxvQSx/GQ+5Gv4/QbQPOLvyLUbMu2mMxgicKYXuTk5JN7zrcJtxyg5V/3Ew62ux2SMUfNEoUxvcxbPIqcuV8mWPk+zS/eSziYGcM7zbHLEoUxSeCvOIXsU79AcNsqWhb9mnCo+wfgjElVliiMSZKsCWeSfdLnCGxeQcvi31ozlElb9sCdMUmUNfk8CAVpfeMJmhpqyT3najx5hW6HZUyPJDVRiMhlwE2AH/iFqt7bxXkXAL9S1dHR7QHAY8AYoAa4VFV3JTNWY5Ila+o8nPxBtLz0EE1P30buedfgLR7ldljGJCxpTU8iUg7MB04FpgFXiciETs4bAtzNoXOJ/RhYoqrHAw8A9yQrTmP6gn/sieR98gYAmp75Ma2rnjuiNbeNcUMy+yjOBhapap2qNgJPAhd3ct6DwG2H7buASI0C4HHgoyJiE+6btOYtHkXehbfgGzGNtuVP0vT0bfZgnkkLyUwUQ4GqmO0qYFjsCSJyDbASeL2ra1U1ABwASpIWqTF9xJNXSO453ybn3KsJN9fT9JfbaV54L8Ha7W6HZkyXktlH4eHQGYgd4OBk7CIyCbgIOIvDEgiHNkN96NruDBqU36NA3VBS0t/tEHpNppSlT8tRcgahybPZ99pf2L/i7zRtepO8itkUTD+X3DFTcTxHN110pvxMwMqSCpKZKHYAc2O2S4HKmO1LgDJgBZAFDBWRJao6F9gZPX+HiPiA/kBtom9cW9uQ0gvSp/N0w4fLlLK4Vo5Jn6DfcWfStvpFmtYspOmDN3FyC/CNnYNvxBS8peNwfNk9umWm/EzAytJXPB4n7gfsZCaKhcCtIlICNBKpPVzVcVBVbwFuARCRUcBL0SQB8DzwBeAnwGeIdGzbIHSTkZycfLJnXUjW9I8T2P4OgQ+W0b72X7Sv/id4fHiHjMVTPArvwGF4Bg7H078YsvslbTW9cDgMwXYItkee/QgFIRiIdL6Hgge/wjGvCQchFCYcDkE4FFm1KPY14ei+cGRxoujrg/sJR9sfwrGBsD8/m7aG1pjjh53T1bJJ4S43juLfJeEzO927r182rY2tvRJLV3wjp+MtGtr79+31O0ap6k4RuRFYTKTG8KCqLheR54EfquqKOJffDDwiImuAfcDnkxWnManC8frwj5qJf9RMwu0tBKs+IFC5lmCV0r52Ee2xD+x5/Th5A3By8nGy8nD8OeDzg+OlJi+Hlpb26Op0ocgf8WAg+sc9AIH2g98JthEOtP07KQTbI+emiOT+We1bdX3xJoE2vLMu7PXbOhm2yMooYLM1PfWdTClLqpcjHAoRPrCb4N6dhBtqCTXuJdy4l3BbE+G2ZmhriswpFQriIRxdutSJLFrt8eJ4fOD1gseH48sCjy+SbHxZke9eP/ii32Nfe3zg9UVfR6/3eKOvI1+O44m8djzR9/MAHvA4kWNOdMyM48S8jtnXEWd0O3Yt7kHF/amtbfj3P8TBWlQCtalDaly9VPtK+DYfPrG4OJ89exo6ObcXef1HVNOMaXoaDWw5/Lg9mW1MGnA8HpwBZXgGlHV7bqonvZ7w5ubjZKfuh76e8PizcXxtbodxRGyuJ2OMMXFZojDGGBOXJQpjjDFxWaIwxhgTlyUKY4wxcVmiMMYYE5clCmOMMXFl2nMUXog8PJLq0iHGRGVKWTKlHGBlSVWpWpaYuDqdjTLTnsw+FVjidhDGGJOm5gJLD9+ZaYkiG5hNZC2LoMuxGGNMuvASmc37TTqZYivTEoUxxpheZp3Zxhhj4rJEYYwxJi5LFMYYY+KyRGGMMSYuSxTGGGPiskRhjDEmLksUxhhj4rJEYYwxJq5Mm+sprYnIKOAmYB+wSVXvczeiIyciXmAh8H1VXeF2PEdKRE4DrgQc4GVVfdDlkHpMRIqBe4BG4G+q+ozLIR2xTPh5dEin3xGrUaSWa4GtQAGwzOVYjtYNQKXbQfSCAcDXgS8Cn3Q5liN1DfALVb0KuMrtYI5SJvw8OqTN74jVKFwkIt8HzovZFQJ+DWwBHgM+5UJYPdZJOe4H1tDFTJSprJOynEvk0+udRD6Vp6NSYIfbQfQGVX02+kk8nX8eiMhFpNHviCUKF6nqT4GfdmyLyCNAA9AEtLkUVo91Uo7Hgf3ALGAscIVLofVYJ2UZAPwcuDfVmwfi2E5kwrcqtwM5Whny8wC4mDT6HbFEkVo6/kjVAQ+5HMsRU9XPAYjIrcBz7kZz1P4HGAZ8V0S2q+r1bgd0BB4Efioi7URqrOksE34eafc7YrPHJoGIdPQxfExVt0T3XUako9pPpL34XvciTEymlAMyqywdMqlMmVKWTCnH4awzu5eJyIlEFv6oiNlXDswnsrDSNOAqEZngToSJyZRyQGaVpUMmlSlTypIp5eiMJYre9zXgWxw6muFsYJGq1qlqI/AkkTbKVJYp5YDMKkuHTCpTppQlU8rxIdZH0ctU9UoAEYndPZRDOxKrgBP6MKwey5RyQGaVpUMmlSlTypIp5eiM1Sj6hgeI7QxyiAyFTTeZUg7IrLJ0yKQyZUpZMqIclij6xg4iwxM7lJImD9ocJlPKAZlVlg6ZVKZMKUtGlMOanvrGQuBWESkhMo3CRaTnE7KZUg7IrLJ0yKQyZUpZMqIcVqPoA6q6E7gRWAysAhao6nJ3o+q5TCkHZFZZOmRSmTKlLJlSDnuOwhhjTFxWozDGGBOXJQpjjDFxWaIwxhgTlyUKY4wxcVmiMMYYE5clCmOMMXHZA3fGJFl0RbbvAJcR+Z3LAv4K/FBVW92MzZhEWI3CmOS7HzgJOEtVpwGzASGyoJAxKc8euDMmiURkFJG1kctU9UDM/lLgFFV9yq3YjEmUNT0Zk1wzgTWxSQJAVXcBliRMWrCmJ2OSK4T9npk0Z/+BjUmuN4DjRaR/7E4RKReRv4lIrktxGZMwSxTGJJGqVgKPAQ+LSAFA9Pt9QK2qNrsZnzGJsERhTPJ9E1gLLBORVURqGWuBK12NypgE2agnY4wxcVmNwhhjTFyWKIwxxsRlicIYY0xcliiMMcbEZYnCGGNMXJYojDHGxGWJwhhjTFyWKIwxxsT1/wEaiIkNar9SFQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ce_losses = [\n",
    "    get_train_and_validation_ce_loss(C, 2) for C in 10**np.linspace(-7, 5, 100)\n",
    "]\n",
    "plt.semilogx(10**np.linspace(-7, 5, 100), ce_losses)\n",
    "plt.legend(['train', 'validation'])\n",
    "plt.ylabel('Mean Cross Entropy Loss')\n",
    "plt.xlabel('C')\n",
    "plt.savefig(\"cross-entropy-loss-vs-c-degree-2.png\",\n",
    "            dpi=300,\n",
    "            bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEQCAYAAACugzM1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU1fn48c+9s2QnCSFAwr4+7PviAuKGC+51adW6VFv119butrZqtVrUr9WuLq1bbWvRWqytVtSWgigiyA4CHpB9C4SEELLO+vtjJjjEZJhAJjOZPO/Xa5jcdZ6TIfPMOffcc6xgMIhSSinVHDvRASillEpumiiUUkpFpYlCKaVUVJoolFJKRaWJQimlVFTORAfQytKAicBewJ/gWJRSqr1wAEXAUqC+8cZUSxQTgfcTHYRSSrVTU4GFjVemWqLYC3DwYDWBQPLeH1JQkE1ZWVWiw2gVqVKWVCkHaFmSVTKXxbYt8vOzIPwZ2lhcE4WIXAPcDbiAXxtjnojYNgZ4IWL3QuCgMWaEiPQGXgS6Aga41hgTy2/YDxAIBJM6UQBJH19LpEpZUqUcoGVJVu2gLE022cftYraI9ABmAlOAMcAtIjKsYbsxZpUxZowxZgxwCnAQuC28+UngSWPMEGAZcE+84lRKKRVdPHs9nQ3MM8aUG2OqgdnAFc3s+2NggTFmoYi4gNPC+0Oo1nFlHONUSikVRTybnoo5ur1rLzCp8U4ikgvcAowMr+oCVBpjfBHH9WzJCxcUZLc42LZWWJiT6BBaTaqUJVXKAalRlkAgwP79+ykvP4jf3/47Me7fn+gIwOFw0LlzPl26dMG2Y68nxDNR2EBkg5wFBJrY78vAP40xDb/GxsfRzHHNKiurSuq2wMLCHEpLDyc6jFaRKmVJlXJA6pSlvHw/TqdNbm4hDocTy7ISHdIJcTptfL4WfZS1qmAwiN/vo6KigoMHD9O5c9cj22zbivoFO55NT7sI9ctt0B3Y08R+lwIvRyzvB3JFxBFeLmrmOKVUCvN46sjP74LT6Wr3SSIZWJaF0+kiL68Aj6euRcfGs0YxF7hPRAqBauByQk1MR4iIBYwHPmxYZ4zxisj7wBeBWcD1wFtxjPOIQ7u2UF9Zjm3bWLaFw7KxHTYO28J2OHA6HNgOGywLrMhnG2wby3KAbYPlANuBZTvA4QTbGVrW/+xKtUAQy7JpYYOCOobQ77RlLS5xSxTGmN0ichcwH3ADzxpjPhKROcBPjTHLCHWJ9RhjGqe3rwN/EpG7gR3A1fGKs0FNVRW8+QAZ1ud/gYHww/e5LS0TtBwEHS6qXG6CtgvblQZON5YrDZxpWO4MLFcGuNOx0rKx0jJDzxk5WOmdws/Z4TdaKaXahpViExf1BbYe7zWKzZ9soqqijGAgSMAfIBAMEggEQj8HgvgDfgJ+f+hnvz/08DU8+z5b9vkI+Lz4fT4sAjjx47Q+e3bhw2X5SbP9ZDoDZNp+0mwf6ZYXNx6c/nrsYDNpyXZiZeVhZ3XGyumC3alr6JFfjJ1XjOV0n9AvsKVSpT08VcoBqVOWkpLt9OzZL6Ht+pGqqqqYOfM+Hnro0Zj2/+ST9fzzn69y552h3v2JvkYRqaRkO9279zmyHHGNoh+wrfH+qXZn9gkZMGQQMKjVzhcMBvH6AtTU+6iu81Fd66W61ovldLBn/2EO1HiorPZwqNrDoSoPFYfrqa4LJQgnfjKtejJtD90zfXTPCtA13UsXdx15Vg2Z3sO49nwCmz7kSDXSsrA6dcXRpR+ObgNwdB2A3aU3lq1vs1In6vDhSjZtMjHvP2TIMO68c9ixd2wH9BMkjizLwu1y4HY5yMtOO7I+2jc+j9dPRVU95ZX1lFXWUVZZx4GKOrZW1PJReS3llZ+N12VZUJTvZki+j4HZNfRwVZDnO4C/ZCO+zYtDO7nScfYYjqPnCJx9xmBn5ce1zEqlql//+hccOFDKj3/8A7Zv30pubh5paWnMnPkIDz30AKWl+zlwoJQJEyZx5533sHLlcp5//mkef/xpvvnNWxg+fASrVq2kouIg3/nOHZx88qmJLlLMNFEkGbfLQdf8TLrmZza5vd7rZ//BWvaWVbPnQDW7S6v5uLSKeZsDhDqWdaeoYBKjezgYkVNBT98O/HvX4du2nPqFf8ZRPATXwJNx9p+I5c5o07IpdSI+WLuXhWuaHIrohE0ZVcSpI4ui7vOd79zB7bffyre+9T2uvPJi/v7331FUVMx///s2gwYN5uc//z+8Xi9f/vKVGPPJ5473+bz84Q9/ZOHC93jmmac0Uaj4SXM56NU1m15dj+7zXFPnZVvJYbburWTjzkO8ayp42+PAoh8Dikdz8qAAo53bsHYvo+695+HDWbiGno57xHTs7IIElUap9ik/vzNFRcUATJ9+HuvXf8wrr8xi27atHDp0iNrams8dc9JJpwDQv/8ADh+ubNN4T5QmihSRme5iWN/ODOvbmQtOBn8gwI59VazZXMaqTw/wlyVV/IUuDCi+kulDfQyrXYl37X/wrv0vzkEnkTbxCm2WUknt1JHH/tbfVtLSPmtKnj37Zd59dx4XX3wZV1wxia1bN9NUJyG3O3SMZVlNbk9mmihSlMO26VfUiX5FnbhkSj/KK+tYsmEfi9aW8PsPaklzDefcYZM4PWM9bH4P35aluEdfgHv0eVjOtGO/gFIdjMPhaHIokaVLl3DxxV/gnHPO45NP1rNp00YCgUCLhshIdpooOojOndI5f3IfzpvUm20lh5m7bCf/XrOff9OLs4fcxPnuZXiWv4Z34/tknHErju6t1/tLqVTQuXMB3bp158EHf3bU+quuuoZHH32IF1/8I1lZ2YwYMYq9e/fQo0eLhqhLanofRQIkSz/3skN1/GfpTuav3I1twZdGBBhfPgeqy3CPuwT32ItCd5dHkSxlOVGpUg5InbIk230UJ6o930eROnUj1WIFuelcffYgZn5tMqMGduHPq+DhiguoKRqPZ/k/qf33/xGoa/8fOEqpE6OJQlGYl8HXLx3BndeOw+9I5ycfD2N1t0vwl26h5l8/J1BZmugQlVIJpIlCHTG4Vx4/+8okpo0p5vkNubzMRQRqD1PzrwfwH9iW6PCUUgmiiUIdJc3t4PrzhnD75SNZVZnH41Xn48NBzRsP4y/dmujwlFIJoIlCNWnsoEJ+/OXxHLQ782Dp2XgdGdS+9UsCh0oSHZpSqo1polDN6lmYzV3XTyC7c1ce2TcNrz9AzZxHCVQfTHRoSqk2pIlCRZWXncYPrxlLp6Je/Lb8dPw1h6l96zGCntpEh6aUaiOaKNQxpbudfOeK0VDQl2cqp+Ev303dwj+1u2EIlGorM2fex5w5b3DgQCk/+MG3mtxnypQJUc+xZ89uHnrofiA0t8XDDz/Q6nHGShOFiklmupPvf3EMhzoN4J36Mfg+XYzXvJfosJRKal26FPLoo789rmNLSvaye/cuoGFui3taM7QW0SE8VMyyM1x8/4tjeOCFegYF9zPggxepHzwCbB19VsWfd+MHcfty4pLTcA2OPuz3T35yB+eccx6nn34WADfd9GVuv/27PP30k9TX13H4cBXf+tZ3mTr19CPH7N27h9tvv5XZs99gz5493HvvXdTW1jJ8+Igj+5SW7uehhx6gquowBw6UMmPGRXz1q7fxm988yp49u3nssf/jjDPOOjK3xY4d23nkkZkcPlxJenoG3/nODxg6dDgzZ95HVlY2xmzgwIFSbrzxq1xwwcWt8vvRGoVqkbzsNL5+2Wj+VHkqNQEX+/7xGEFv/bEPVKqdO/fcGcyd+w4AO3fuwOPx8Oqrf+POO+/h+ef/yp133s0zzzzV7PGPPfYwM2ZcxAsvzGLkyNFH1v/3v+8wffq5PP30C/z5z3/jlVdeoqKigm9/+weIDOX73//RUed54IF7uPLKL/GnP73M7bd/j7vv/hEejweA/fv38eSTz/Lww7/kiSd+02pl1xqFarEBPXK5dPponptbwe3B/1C//DXST/pSosNSKc41+NRjfuuPp1NOmcKvfvUINTXVzJ37Dueeez5XXXUNixa9z/z5c1m3bi21tc138lixYjn33jsTgHPOOf/INYdrrrmOFSuWMWvWX9i6dTM+n5e6uqbPU1NTw65du5g27UwARowYSadOndixYzsAkyZNxrIs+vcfQGXloVYru9Yo1HGZNqYHxcPHsahuIJ61/8FftiPRISkVVy6Xi1NPncrChe8xb95/mT79PL7xja+xYcM6RIZw/fU3Re3gYVnWkcFKLcvCDg+4+bvf/Yq///1luncv4oYbbiY3N6/Z8wSDnx9UMBjkyPDnkXNetKa4JgoRuUZE1ovIJhH5RhPbRUTeFZHVIvKOiOSH198gIntFZFX4MTOecarjc+30QazInkZNwE3NghcIBpJjZEyl4uXcc2fw8ssvkpubR2ZmJjt3bufmm2/jpJNO5f33FxCI8jcwceJk3nlnDgALFszD4wk12S5btoRrrrmOM888mx07tlNaup9AIIDD4fzc/BdZWdkUF/dgwYJ5AHz88VrKy8vo339AnEocEremJxHpAcwExgP1wCIRmW+MWR/ebgGvA982xrwtIg8DdwI/AiYA3zPGvBSv+NSJczkd3Hb1ybz0+41cd+B9vBvm4x5+VqLDUipuRo0aQ1VVFZdeegWdOuVy4YWXcN11V+F0Ohk3biJ1dXXNNj99//s/4r777ub1119jyJChZGZmAfDlL9/IAw/8lLS0NLp27c6QIcPYs2c3gwcLVVWHeeCBe7jggkuOnOenP32AX/ziQZ577g+4XG5mznwEl8sV13LHbT4KEbkBOM0Yc3N4+R7AMsbcH14eDzxjjBkXXu4E5BljdojIYqAC6AGsBm43xsRyO3BfdD6KNlVYmMPv/76SPuueY1D6QXKufhg7My/RYbVYqr0nqVAWnY8iflo6H0U8L2YXA3sjlvcCkyKWBwIlIvIcMBbYANwese+jwCLgQeBx4NpYXzhc4KRWWJiT6BBazc2XjeJnW89koGcWgbVv0u3C2xId0nFJpfckFcqyf3+oZdzpTJ1LqclSFtu2W/R/JJ6JwgYiv9ZbQGQ6dQKnE6p1LBORB4BfAjcaYy5r2ElEHgE2t+SFtUbRdgoLc6g4WMMl50/mw1eWcsrq/7Fv6HTsTl0THVqLpNp7kgplaWjvT5Zv4ScqmWoUgUDgqP8jETWKJsUzve0CiiKWuwN7IpZLgE3GmGXh5ZeASSKSKyLfjdjPAnxxjFO1gr7dO3Go/zn4ghYVH/w90eGolGA12ctHnZjQ77RlvaLimSjmAmeJSKGIZAKXA29HbF8EFIpIw50nFwHLgSrghyIyObz+m8BrcYxTtZIZZ4xikXcYzp1L8ZfvTHQ4qp1zu9MpLz+Az+fVccVaQTAYxOfzUlFxALc7vUXHxq3pyRizW0TuAuYDbuBZY8xHIjIH+Gm4ueky4BkRySJUA7nOGOMXkauAp0QkA9gIXB+vOFXr6ZTlJn3shdSu/YTad1+i2xd+mOiQVDuWn18I1FFauo9AwH/M/ZOdbdtRu8+2TQwOMjKyyc7ObdFxcev1lCB90V5PbapxWby+AG89+xRn2EtJu+gnuIsGJzC62KXye9KeaVnaxrF6PcXU9CQiOeHnsSJynYjEt9OuardcTpve0y6lOuCmZKG2GCqVCo6ZKETkfuAPItKb0DWGrwDNj3ylOrzRQ4pZ5xpFXvkGPOV7jn2AUiqpxVKjmAF8ldDF6JeMMWcCo6Mfojoyy7IoOuVCfNjsWqC1CqXau5ianowxNcDZwLzwqrS4RaRSwtAhfTDOIeTtX4G3qiLR4SilTkAsiaJMRJ4kNP7S3PCYTNqeoKKyLIv8SRfhsvxsefdfiQ5HKXUCYkkU1xNKDBeEaxZB4Ia4RqVSgowYwha7H3m7F+Gtb36cfqVUcjtmojDG7AN+E77vYSywHiiPe2Sq3bMsi8yxM8i06tm44O1jH6CUSkra60nF1YCx49lPF9K2LUz4zUZKqeOjvZ5UXNm2ja//FLpRxqbVqxMdjlLqOGivJxV3/U+djifopHLV3ESHopQ6DtrrScWdKyOLss4j6ecx7NpdmuhwlFIt1JJeTzO015M6XkUnnU+a5ePThf9NdChKqRaKtdfTX4ARIvJV4LnwOqVilt1zEBXubnQvX8qhw3WJDkcp1QKx9Ho6F1gGXApcDCwVkUuiH6XU0SzLInP4GfRwHGT5hx8lOhylVAvE0vT0ADDNGHOZMeZiYApwX1yjUimp8+hpeHFibf4Av3aVVardiCVRuI0x6xsWjDHrAEf8QlKpynJnUNttFMOsraw1JYkORykVo1gSRa2ITGhYCP9cE7+QVCorHHs6mbaHLcsXJToUpVSMYpkK9YfAv0VkE6EeT0OAK+MalUpZrp7DqXRk061iNaUVF1CYl5HokJRSxxBLr6f3gWHAg8AjwFBjzIJ4B6ZSk2U7cA48iWGu3Xy44tNEh6OUikEsNQqMMeXAWw3LIrLQGDMlblGplJYzfCo1Zi6HNyzGN204TkdMAwQopRLkeP9CR7VqFKpDsQt648nqxgg2smKj3qmtVLKLqUZxvETkGuBuwAX82hjzRKPtAvwByAdKgC8ZYw6GR6p9EegKGOBaY0xVPGNVbceyLLKGTaX/0tm8vHIDk4Z2S3RISqko4lbnF5EewExC912MAW4RkWER2y3gdeBhY8xoYCVwZ3jzk8CTxpghhG72uydecarEcA86mSCQu38l5ZV6p7ZSyazZGoWI/LaZTRbgjuHcZwPzwtc3EJHZwBXA/eHt44BqY0zDjDYPAnki4gJOI3QnOMALwALgRzG8pmon7OwCAl0GMqZkGx+uK+GCk/smOiSlVDOiNT2VRdn2UAznLgb2RizvBSZFLA8ESkTkOWAssAG4HegCVBpjfBHH9Yzh9Y4oKMhuye4JUViYk+gQWs3xlsU9/nQc7zzLnHUb6HLRCCzLauXIWkbfk+SkZUm8ZhOFMeZnJ3hum9B9Fw0sIHLcBidwOnBaeJrVB4BfAnc1Oo5Gxx1TWVkVgUDjUySPwsIcSksPJzqMVnEiZQkUjiCIRXH1Bhav3s3AHrmtHF3s9D1JTlqWtmHbVtQv2PHsl7gLKIpY7s7R81iUAJuMMcvCyy8RqnHsB3JFpGGYkCJ0/ouUZGfmYXUbxDj3dhau2XvsA5RSCRHPRDEXOEtECkUkk9BUqm9HbF8EFIpIw7SqFwHLjTFe4H3gi+H11xNxD4dKLWkDJ9PNcYjt5hPqvf5Eh6OUakIsw4wfVxdaY8xuQs1I84FVwCxjzEciMkdEJhhjaoHLgGdEZB1wJvD98OFfJ9RLaj0wlVAXW5WCnP3GE8RimLWFlXpPhVJJKZYksF1EXgCeNsZsb8nJjTGzgFmN1s2I+HkJR1/gbli/ndD1C5Xi7Mw8HEXC+D07eG3tXk4a3j3RISmlGoml6ekkwAe8JyL/FpELwvdAKNUqXAMmUWhVUL5zC4eq6hMdjlKqkVgGBdxpjLkX6Ac8A/wO2Coid4hIWrwDVKnP2TfU/DTGvZ2PNuxPdDhKqUZiupgtIkOBXwC/J3S94XagL/BK3CJTHYadmYuzSJiQsZMP1+mERkolm1guZi8E5gG1wERjzBeMMW8QShZT4xyf6iCc/cbThYNU799NSbnOi6VUMomlRvEk0NsYc7cxZlfDSmNMAOgdt8hUh+LsOw6A0e4dLNZahVJJJZZE8QrwNRH5l4j8Q0S+0rBBR3RVrcXOLsAu7Mek7N18uK6EYDB576xXqqOJJVH8BrgK+BcwB7hJRH4e16hUh+TsO45ugX14DpWzZU9losNRSoXFkijOAaYbY543xjwLTEfnzFZx4Ow3HoAx6TtZvG5fgqNRSjWIJVGUcvSNeQGgIj7hqI7MkVeMnVfEyZ32svSTffgDLRoLUikVJ7Hcmb0KeD98d7aP0BhMB0TkewDGmF/GLzzV0Tj7jqf7qjn4ag+zYftBRvQrSHRISnV4sdQoMoC1wHhgMrCN0AivI4ERcYtMdUjOfuOxCDA2cw9L1mvzk1LJ4Jg1CmPMVwBEpA/gMsZ8GveoVIdld+mLldWZU9NL+N3GUq4/14/L6Tj2gUqpuInlhruB4dFdVwHLRWSziAyJf2iqI7IsC2ffsRR5tuOrr2fN5vJEh6RUhxdL09PjwCPGmHxjTC7wc0I34SkVF84+47ADXsZk72fJBm1+UirRYkkU3Ywxf2pYMMb8ESiMX0iqo3MUC7gzmNp5P6s/PUBtve/YByml4iaWROEUkc4NCyLShc/Paa1Uq7FsJ85eo+lZvxmfz8eqTQcSHZJSHVos3WN/BywWkb8RShBfAn4V16hUh+fsOw7f5sWM7lTBkg37OHmETmikVKLEUqP4I3Ar4AYyga8bY56Ka1Sqw3P2Ggm2g2ldDrBuazlVtd5Eh6RUhxVLjWKpMWYMobmvlWoTljsDR/FQ+hzcjD8wiOVmP9PG9Eh0WEp1SLHUKKpFpGfcI1GqEWffcTiqSxneuV5vvlMqgWKpUWQRmvp0J3BkWHFjzKi4RaUU4OwzlvqFf+bMwgM8btKpqKonL1tn31WqrcWSKL59vCcXkWuAuwEX8GtjzBONtt8L3AQcDK96xhjzRHPrjzcO1T7ZWfnYhf3o59tMkJ4s3bCf6RN7JTospTqcWBLF9caYmyNXiMhsYEG0g0SkBzCT0BhR9cAiEZlvjFkfsdsE4EvGmA8bHd7cetXBOPuMJbDsHwwptPhowz5NFEolQLOJQkSeAnoAU0Uk8gY7F9A/hnOfDcwzxpSHzzcbuAK4P2KfCcBPwuNIvQf8wBhTF2W96mCcfcfiWfYPzu5ezuNrg5RW1FKYl5HosJTqUKLVKJ4jNDrsaODViPU+YHEM5y4G9kYs7wUmNSyISDawErgD+BR4AbhHRB5qaj1wVwyvCUBBQXasuyZMYWFOokNoNfEsS7DLUHbmdWWIYweQz7odFVw5qGtcXkvfk+SkZUm8ZhOFMWYZsExE5hpjdh3HuW2OvoPbIjTpUcP5q4AZDcsi8hjwvDHmrqbW04JEUVZWRSCQvDePFxbmUFp6ONFhtIq2KIvVcwy+DfMYUjyZeUt3cPqoolZ/DX1PkpOWpW3YthX1C3Ys3WN7ich8EVktImsaHjEctwuI/IvuDuxpWBCR3iJyU8R2C/A2tz6G11Mpytl3LPh9nFVUya7SanaVVh37IKVUq4nlYvYfCDX/rKBlYzzNBe4LX9+oBi4HbonYXgs8IiLzCU2G9A3gtSjrVQfl6D4Y0rIYxDZsS1iyfh89pyV/86JSqSKWROE7nulOjTG7ReQuQnd0u4FnjTEficgc4KfGmGUicivwRnj7QuAxY4ynqfUtfX2VOizbgbP3aHw7VjO8zySWrN/HF07rj2VZiQ5NqQ4hlkTxsYiMNMasbenJjTGzgFmN1s2I+PlVjr5QHnW96ricfcbi27SIMwbU8ttt9WzeXcnAnrmJDkupDiGWRNGf0Mx22wk1CwF6Z7ZqW85eI8HhZGBwKy5nTxavL9FEoVQbiSVRxNzbSKl4sVzpOIqHEdi1mjEDRvPRhv186axBOB2x9MdQSp2IaDfc9TbG7DDGfO4ObBE5L75hKfV5zr7jqH//BU6TAEuNl/XbDjJqQEGiw1Iq5UX7OvbPhh9EpPH1ggfjE45SzXP2GQtY9PdvISvdyZL1JYkOSakOIVqiiOxS0njIDu1uotqcnZmLo9tAAjtWMl66smLjAeo9/kSHpVTKi5Yogs383NSyUm3C2W8cgbIdTO3vot7rZ/nG/YkOSamUF2uNQqmk4Ow7HoCe9Z/SJTedRR9r85NS8Rat15MtIvmEEoYj4mcAR9wjU6oJdqeu2J174t++glNGfJE3PthGeWUdnTulJzo0pVJWtBrFSOBA+DESKItYHhH/0JRqmrPvOPwlGzllUBZBYLFOk6pUXEUbPVY7qKuk5Ow7Hs+K18k/tJGBPXNZ9HEJ50/urUN6KBUnmgxUu2MX9MbKLsC7dTmnjOjOngPVbN+XnMM3K5UKNFGodseyrFDz0+6PmTCgE06HzaK1elFbqXjRRKHaJWf/ieD3kbZ/PWMGdWHx+n34/IFjH6iUarEWJQoRcYtI73gFo1SsHN0GYmXk4tuylCkju1NV62X1pwcSHZZSKemYiUJELhOR34lIDrARWC0i345/aEo1z7JsnP3G49u5huG9ssnPSeO91XuPfaBSqsViqVH8GHia0Ax1HwK9geviGZRSsXD2nwg+D4HdHzNlZBEfbymjvLIu0WEplXJiSRRWeNKis4G3jDGHYzxOqbhydB+MlZ6Db8sypo4KTc++cI3WKpRqbbF84AdE5CrgXOA/IjID0KuGKuEs24Gz7zh8O1ZRkO1gWN983l+zh0BAhyJTqjXFkii+D9wC/MQYU0JoIqNvxTUqpWLk7D8RvHX4d61j6uhiyirrWb+9PNFhKZVSjjnDnTFmIaFmJ0TEDVxtjNkR78CUioWjeAikZeHduoyxU28mO8PFe6v3MqKfTmikVGvRXk+qXbNsJ84+Y/FtX4HT8nPKiO6s3FhKZbUn0aEplTJimTP7x8DNfNbr6VZgHvCbYx0oItcAdwMu4NfGmCcabb8XuAk4GF71jDHmCREZAzwLdALeA24zxvhiKpHqcFz9J+HbuBD/ro+ZNmYw/1m6k/dW7+HCU/omOjSlUsLx9HqqjOU4EekBzASmAGOAW0RkWKPdJgBfMsaMCT8aEsmLwDeNMYMJDW3+tdiKozoiR89hoeanzUsoKshiWN985q/cjT+gfS6Uag0t6fV0Hi3r9XQ2MM8YU26MqQZmA1c02mcC8BMRWSMij4tIuoj0ATKMMYvD+7wAXBlLYVTHZNlOXP0m4tu2kqCvnrPG9eTg4XpWbSpLdGhKpYRYmp6+D9wH/NgYUyIisfZ6KgYiO7XvBSY1LIhINrASuAP4lFBCuAf4dxPH9Yzh9Y4oKMhuye4JUViYk+gQWk0ylKV2/Ons/eRdMis2ctbJJ/Py/E95f+1ezpvSeLr35iVDOVqLliU5tdeyxNzrSUT6iMhAY8ypMZ7b5ui5tS0iaiLGmCpgRsOyiDwGPA/MiUZOvdwAABvkSURBVHZcLMrKqpK6L31hYQ6lpakxLHaylCWY0RsrM4/yFe+S0WUkp40q4tUFW1i9oYTiLlnHPD5ZytEatCzJKZnLYttW1C/YsVxrGCQi64BVwHIR2SwiQ2J47V1AUcRyd2BPxHl7i8hNEdstwHus45RqimXbOPtPxLdzNUFPDVNHF+N02MxbsSvRoSnV7sVyjeJ3wCPGmHxjTC7wc+DJGI6bC5wlIoUikkmo19TbEdtrgUdEpJ+IWMA3gNeMMduBOhFpqLlcB7wVY3lUB+YaMBn8PnzbVtIp082koV354OMSauu1w5xSJyKWRNHNGPOnhgVjzB+BwmMdZIzZTegu7vmEaiOzjDEficgcEZlgjCkl1NX2DcAQqlE8Fj78WuBXIvIJkA38tgVlUh2U3XUAVk4XvJuXAHD2hJ7Ue/wsWKUVUqVORCwXs50i0tkYUw4gIl04+hpCs4wxs4BZjdbNiPj5VeDVJo5bTcSFb6ViYVkWrgGT8ax+i0BtJX27d2JI7zz+u2wnZ0/oidOhY1kqdTxibXpaLCIPiMj9wCLgqfiGpdTxcQ48GYIBfOFaxXmT+3DwcD1L1u9LcGRKtV+xJIo/EmoicgOZwNeNMZooVFJydO6JXdAH76ZFAIzs35kehVm889EOgsHk7QmnVDKLpelpqTFmDKFrDUolPdfgU6n/cBb+g7tx5PfgvEm9ee7NDXy8tZyR/XWwQKVaKpYaRbWItOiGN6USyTnwJLBsfBs/AGDysG7k56Tx1uLtCY5MqfYplhpFFrBVRHYCVQ0rjTGj4haVUifAzuiEo9dIvJ9+iHviFTgdNtMn9OKV+Z+yZU8l/Ys7JTpEpdqVWGoU3yY0btNXgNsjHkolLdfgUwlWH8S/Zz0A08YUk5Xu5PUPtiY4MqXan6iJQkQKgA3GmAXGmAWAA1gf/lmppOXsPQbcmXjDzU8ZaU7Om9ybNZvL2LKnMsHRKdW+NJsoRGQ48AkQObbTF4A1IiLxDkypE2E53bgGTMK3bTlBTy0AZ47rSXaGi38t1FqFUi0RrUbxMPBtY8xrDSuMMd8kNJHRI/EOTKkT5Ro8BXyeI3dqZ6Q5OXdSL9ZuKWPz7kMJjk6p9iNaougdvrP6KMaYF4ABcYtIqVZidx2And8D7yeftZSeNV5rFUq1VLRE4Y+yTSckVknPsixcQ6YRKN2Kv2wHAOluJ+dP7s3HW8vZtKsiwREq1T5ESxT7wnNXH0VExgLV8QtJqdbjGnQKOJx4N3xWqzhzXE9ys9y8Mv9TvVtbqRhESxQPAP8SkZtEZIiIDBORm4F/Ave3TXhKnRgrPRtnvwl4P11E0FcPQJrbwWWn9Wfz7kqWm9IER6hU8ms2URhjFhGaC+LLwEeEBgO8ArjWGPPftglPqRPnGjINPLX4tiw7sm7KyCJ6dMli9rub8flbNIGiUh1O1DuzjTHvAWe2USxKxYWjaAhWbje8nyzANTjU29u2La48YyC//vtq5q/czTXn5yY4SqWSlw7Qr1KeZVm4h0zDX7IR/8HdR9aP7N+ZYX3zeX3hVqpqvQmMUKnkpolCdQjOwVNCF7XX/e/IOsuyuOqMgdTU+Zj1zicJjE6p5KaJQnUIdkYnnAMm4934AUFPzZH1vbvlcPrYHry5cAs79h1OYIRKJa+YEoWI9BGRsSIyruER78CUam3u4dPBV4/XLDxq/Rem9Scny81f3jEEtLusUp9zzEQRnv50A/AaofmtXwVmxzkupVqdo7AvdreBeNb9j2Dws55OWekubrpoOJv3VLJwzd4ERqhUcoplPorrgIHGmD3xDkapeHMPP5u6eb/Hv3Mtzt6jj6w/Y3wv3nx/C3+f/yljB3UhJ9OdwCiVSi6xND3tPN4kISLXiMh6EdkkIt+Ist8FIrI1YnmaiJSJyKrw44/H8/pKNebsPwErMw/PurlHrbcsi+vOFeo8fl7636YERadUcoqlRvE/EXkE+BdQ27DSGLMi2kEi0gOYCYwH6oFFIjLfGLO+0X7dgEcBK2L1BOBRY8xDMZVCqRhZthPX0DPwLH+NQMVe7LyiI9t6FGZzwcl9eP2DbUyQrowbXJjASJVKHrHUKG4ErgRepGXXKM4G5hljyo0x1eFjrmhiv2eBnzVaNxE4R0TWiMjrItIrhtdTKiauoaeDw4ln7Tuf23bhKX3p3TWbP7/9CYdrdOxLpSCGGoUxpt9xnrsYiLwyuBeYFLmDiHwLWAEsbnRsBfCKMeYfInIb8DJHT6AUVUFB9nEF3JYKC3MSHUKraX9lyaF01BlUrXmX/HOuw5mdB3xWjh9cN4Hv/XoBr763lTuum5DIQI9b+3tPmqdlSbxjJgoR6ULognY2oeYhB6GL29ce41AbiOxraAFHupqIyAjgcuAsoGfkgcaY2yJ+/r2IPCwiucaYmGabKSurIhBI3m6OhYU5lJamRp/99lqWwKCzCK6cS8l7/yRt4uVHlSPbZXPRKX157f2tjOibz4QhXRMcbcu01/ekKVqWtmHbVtQv2LE0Pb1CqBnpZqAXcAMRH/hR7AKKIpa7A5EXxa8Mb18GzAGKReR9EbFF5C4RcTQ6ny+G11QqJnZed5z9xoe6ynpqP7f9/JP60K8ohxfe+oQDFZ/frlRHEkui6GOMuYDQh/njhJqAhsRw3FzgLBEpFJFMQrWHtxs2GmPuNcYMNsaMAWYAe4wxU40xAeCy8P6IyPXAkvB1DqVajXv0+eCpwfvJe5/b5nTY3HrJCIIE+cPr63SEWdWhxZIoSsLPm4ARxpjdgOtYB4X3uwuYD6wCZhljPhKROSJyrIbfG4DviMg64CvAV2OIU6kWcXQdgKNI8Kx9h6D/8xXWrnkZ3HDeEDbvqeS197ckIEKlkkMs3WP3i8gdwIfAz0SkEsiM5eThObdnNVo3o4n9tgF9I5bXAafE8hpKnQj36BnUvv0rqj5+D4onfm77pKHd2LD9IG8t3sGQ3vmM7F+QgCiVSqxYahS3AvXGmIWErifcD/worlEp1UYcvUZhF/Th4AevEgw0PU381WcNomdhFk+/vo59B2ua3EepVHbMRGGM2Q88IyIjgR8DpxpjXot7ZEq1AcuycI+/BN/BEnyfftjkPm6Xg9svH4VlWfx29hpq67VfhepYYhkU8CRgM/AmoXsjdoqINguplOHsMxZ3t37Ur3i92VpFYV4G/+/SEewrr+Xp19cldfdrpVpbLE1PvyDUPbbMGLOL0D0Vv4lrVEq1IcuyyD/tiwQr9+PbtKjZ/Yb2yeea6YNYvbmM2Qs2t2GESiVWLIkiM3J8JmPMHGK7CK5Uu5E5aAJ2lz7Ur3yj2VoFwBlje3DG2B68vWQH/1m6sw0jVCpxYkkUXhHJJ3yXtYhIfENSqu1ZlkXa+EtDtYqNH0Td79rpgxk/uJCX/7eJxetLmt1XqVQRS6L4ObAA6CkiLwGLwuuUSimO3mOwC/tTv/w1gr7mBwS0bYtbLh7G4F55PPfvDXy8pawNo1Sq7cXS6+nfwBeAe4EPgCnGmFfjHZhSbc2yLNImX0Ww+iCej/8TdV+X08G3Lh9JUUEWv/vHWtZtK2+jKJVqe80mChHp3PAAyoG/Ebp5bl94nVIpx1k8BEfvMXhWvkmgLvoAbpnpLn5w9Ri65Wfw29lr+Hir1ixUaopWozgAlEY8DjR6ViolpU2+Enx1eFa8fsx9O2W6uePqsXTvnMlvZ69lrTZDqRQULVH8mdD9E78ERhpjbGOMo+G5bcJTqu058nvgktPwrp9HoHL/MffPCSeL4i6Z/Hb2Gj5cpxe4VWppNlEYY24ExgCrgd+IyIci8nURyWur4JRKFPf4S8F2UL/4bzHtn53h4odXj2VQz1yeeWM9b364jWBQb8pTqSHqxWxjTI0x5kVjzHRC80fkAvNFJLa/HqXaKTsrH/fYi/BtW45v59qYjslMd/Hdq8YweVg3Xl2whb+8Y3R4cpUSYuke26Aw/OgCaK1CpTz3qPOwcrtRt+hFgn5vTMe4nDZfu2gYM07qw7ur9vDISyupqKqPc6RKxVfURCEivUTkxyKyntA1ixJgsjHm3DaJTqkEshwu0k/5MsFD+/CseSfm42zL4orTB3DbJcPZse8wP/vjUjburIhjpErFV7NDcYjIfEAIdYu91hizss2iUipJOHuNxNl3PJ4Vr+MadDJ2duzzUUwa2o3iLlk8/o+1PDJrJRee0oeLTu2Lw25JRV6pxIv2P3YaoSamrwILRKQy/DgcnrxIqQ4h7eSrAahb+JcWX6DuWZjNT2+YyORhXXn9g208+JcV7CvXOS1U+xItUfQDhgIjgJERj4ZlpToEO6cLaRO/gH/HqmbnrIgmM93J1y4azm2XDGf/wRruff4j5izerhe6VbvRbNOTMWZ7WwaiVDJzjTgH79Zl1C36K44ew7AzW96fY9LQbgzqmceL/zHMfnczH35cwnXnCoN7ad8Qldy0sVSpGFi2Tca0m8FXT/37fzrueyTyc9K4/fJR3H75SOo8Ph7+6wqeeG0tJdocpZKYziuhVIzsvCLSJlxO/ZK/4du8GNfAk4/7XGMHFTKsT2fe/mgHby/ZwcqNB5g2ppgZJ/WhIDe9FaNW6sTFNVGIyDXA3YAL+LUx5olm9rsAeNwY0y+8nAf8FehPaFypq4wxOi6CSjjXyHPxbltO3ft/xtF1AHanrsd9rjS3g0um9OP0sT1444OtLFi1h/dW7+Gk4d04f3IfirtktWLkSh2/uDU9iUgPYCYwhdBQILeIyLAm9usGPApYEat/DrxvjBkKPINOvaqShGXbZJx5K1hQ+7+nCPp9J3zO3Cw3Xz5HePjWkzljbA+WbtjP3c8u4bG/rWK5KcUf0IveKrHieY3ibGCeMabcGFMNzAauaGK/Z4GfNVp3AaEaBcBLwPki4opbpEq1gJ1TSPppNxEo3Ur90tmtdt6C3HSumT6YR75+CpdO6ceeA9U88dpa7nhyEa/M/5TtJYd1/CiVEPFseioG9kYs7wUmRe4gIt8CVgCLmzvWGOML37dRCOyJW7RKtYCr/0T8w87Eu+ZtnMVDcPYe02rn7pTp5uIp/bjglD6s2VzGe6v28N+lO3l7yQ66dc5k7KAujB5QwIAeuTgd2h9FxV88E4VNeJ7tMAs4UocWkRHA5cBZQM9Gx1pNLMdc/y4oyG5RoIlQWJiT6BBaTaqUpaXlCFz0NfaUbaX+3WcouPFh3AXFrR5T9265nHNKfw7XeFi0Zi8LV+1m7rJQ0shMdzK8fwHD+xUwrF8BA3rm4nY5jqssyUzLknjxTBS7gKkRy905ukZwJVAELAPcQLGIvG+MmQrsDu+/S0ScQA4Q84wwZWVVBALJW0UvLMyhtDT67GntRaqU5XjL4Trz63hfu5/dL80k69J7sNLidwF63IDOjBvQmdp6H+u3HWTtljLMzgqWrt8HgMO2KCrIZHCfzhTkuOneOZPunTMpzMtotzWPVPn/BcldFtu2on7BjmeimAvcJyKFQDWh2sMtDRuNMfcSmocbEekLvBtOEgBzgOuBB4EvErqwHdvwnUq1ITunkPRzbqf23/9H7dwnyTj/u1h2fHudZ6Q5GS+FjJdCAA5Ve/h0VwXbSg6zY18VK81+Dh7+bMRaC8jNdlPQKZ38nDQ6ZbnplOUmJ9NNVrqTzHQnmWku0t2OIw+3y4HDtrCsxpV71RHF7X+0MWa3iNwFzCdUY3jWGPORiMwBfmqMWRbl8HuAF0RkHVABXBuvOJU6Uc7ug0mfeiN1C56j/oO/kjbl+jb9gM3NcjNeujJeQl11Cwtz2LaznJKyGkrKayitqKW8sp6yyjp2H6hmw/aDVNcdu7eWZYHb6cDpsHA6bJwOC4dt43BYOGwL27awLetIQrEtsCwLK/xsW6GTWAAWWIS2hRcjXqf531VampP6et+ReNozt9uJx3PiveSimTq6mDEDu7T6ea0U60XRF9iqTU9tJ1XK0hrlqF/yCp7Vc3CPvYi0iZe3UmQtF0tZvL4AVbVeauq8VNf5qK33UefxU+cJPXt8Abw+Px5vAL8/iNcfwO8P4A8E8QWC+P0BgkHwB4IEAgECQQgGgwSCQDBIAAgGgoQXgSDBYPiiZcSfZpBGf6eNFp1OBz6fv/HqdsnptPH54tvV+ZyJvTh1ZFGLj4toeuoHbGu8Xe/MVqqVuCddSbC+Cs/KN7DcGbhHz0h0SM1yOW3yc9LIz0lLdChRpcoXEWjfZdFEoVQrsSyLtCk3EvTUUb/kFXBl4B52RqLDUuqEaaJQqhVZtk36mbdQ66unfuGfwO/FPfKcRIel1Alpn33mlEpilu0kY/o3cfabQP2Hs6hf9preUa3aNU0USsWB5XCRftb/wyVT8az4F/WLXiSoYzapdkqbnpSKE8t2kHbaTZCWhXfN2wQq95Nx5m1xvSlPqXjQGoVScWRZFuknfYm0qTfi372e6n8+gL9ChyxT7YsmCqXagHvo6WRc+CPw1FDz2v14Ny1KdEhKxUwThVJtxNl9MJmX3YujoDd185+mdt7vCdZXJzospY5JE4VSbcjOLiDjwjtxT/gCvs0fUf3qT/FtX5nosJSKShOFUm3Msm3Sxl1M5iV3Y7nSqX3nN9S8/SsClfsTHZpSTdJEoVSCOLr2J/Pyn5F20hfx7zVU//0n1H3wIoHqg4kOTamjaPdYpRLIsp24R52Pc8BJeJa9hnf9fLwb3sU15DRcw8/Gkd/6kyEp1VKaKJRKAnZWPunTbsI97iI8K9/E+8kCvOvnYXcbiHvINJx9xmKlt93MjcFAADw1BOurCXpqCXpqCHrrwOch6KsHnxcCXoJ+H/h9EPARDPgh4A8v+wkGfKFhxi0H2DYEAwT9fgiEthPwEww23ITY9NwXJeluPB5/OKggEAw9W3bonJYdWg4GGoapDb3OUXfCRw5XG/xsXcN5LAsr/EzkcZbFUQOiN46v8WsEg6HyNHUXfjDAXrcTj9cfOueR12rY1/osDgjHH/x8vEeOsY6UPxR76PfhktNw9hr5+dc/QZoolEoidk4h6afdiHvCZfg2fYD3k/eoW/AcWBZ2YX+cPUfg6DoAu6AXVmZei+a9CHrrCdZUEKitJFh7iGBtJcG6wwRrD4d+rj1EoKaCYG0leGpbGLjjyMOyneBwhpYbPsQD/tCHWXi9ZTvCH9I2oQ/jYGjI8fBnYMM6X61FwOs/6kM9VJjAkYRgWdZnH+qRPx/1IR/xw1EJIAiBQETCOvLSoX+OfOZHfEAfJWKCjYb4rIgW/WDwyOsFAg6CXt9n8VuNWv4b4rAA7M9+D0cm8Wh07oD/s2PCj0B1eRNvzonTRKFUErIzc3GPnoFr1PkESrfg27EG3661eFa8zpFPr7Qs7OwCrIxOWOk5WE4XAKXpbmqrqgl6aqC+hkBdJcGaQ+Cta/rF3BlYGZ2wM/NwFPQJnS8tCys9G8udGdoefuB0YznTwOHCcrjCH/zOuE3U1J6H5m6sPZdFE4VSScyyLBxdB+DoOoC0CZcRrK/GX76LQPlOAuW7CFRXEKyrJHBoH/hDswXX2BYBy4mVlonlzgx9+PfKxcrMxc7MCyWCjNzPEoxDPwZUdPo/RKl2xErLwlkkUCTN7tOev7mq5KTdY5VSSkWliUIppVRUmiiUUkpFFddrFCJyDXA34AJ+bYx5otH2y4CfAQ5gKXCLMcYjIjcADwP7wru+aYy5K56xKqWUalrcEoWI9ABmAuOBemCRiMw3xqwPb88CHgfGGWP2icjLwI3A08AE4HvGmJfiFZ9SSqnYxLPp6WxgnjGm3BhTDcwGrmjYGF7XN5wkMoGuQMMgNxOBG0RkrYi8KCL5cYxTKaVUFPFMFMXA3ojlvUDPyB2MMV4ROR/YCXQB/hOx7wPAqPC2x+MYp1JKqSjieY3C5qhBVrCAz80ub4x5CygQkQeBp4BrjDGXNWwXkUeAzTG+pgOgoKDtxsQ5XoWFOYkOodWkSllSpRygZUlW7aAsjqZWxrNGsQsoiljuDhyZLFhEOovIORHb/wqMEpFcEfluxHoL8MX4mkXH3kUppVQzmvwMjWeNYi5wn4gUAtXA5cAtEdst4EURmWCM2QFcCSwEqoAfisgiY8wS4JvAazG+5lJgKqGmK3/rFEMppVKeg1CSWNrURivY1JC4rSTcPfYngBt41hjziIjMAX5qjFkmIpcC9xNqoloP3GaMOSQiU4HfABnARuB6Y8yhuAWqlFKqWXFNFEoppdo/vTNbKaVUVJoolFJKRaWJQimlVFSaKJRSSkWliUIppVRUmiiUUkpFpYlCKaVUVDpndhIRkb6E5u+oALYYY55MbETHT0QchO7Ov8MYsyzR8RwvETkN+CqhkQQWGGOeTXBILSYiXQjdwFpNaG6XfyU4pOOWCu9Hg/b0N6I1iuTyfWA70AlYlOBYTtRPiBjbqx3LA24FbgAuSXAsx+tbhCYOu4Wjh9Fpj1Lh/WjQbv5GtEaRQCJyB3BuxKoA8HtgG6FBEi9NQFgt1kQ5ngLW0cxIlMmsibKcQ+jb68OEvpW3R90JDdLZ7hljXg9/E2/P7wcicjnt6G9EE0UCGWN+AfyiYVlEXiA0KGIN4ElQWC3WRDleAg4RmqlwAHBdgkJrsSbKkgf8Cngi2ZsHothJaMC3vcfaMdmlyPsBoUnc2s3fiCaK5NLwIVUOPJfgWI6bMeZqABG5D/h3YqM5Yb8lNOHWd0RkpzHmx4kO6Dg8C/xCRLyEaqztWSq8H+3ub0QHBYwDEWm4xnChMWZbeN01hC5Uuwi1Fz+RuAhjkyrlgNQqS4NUKlOqlCVVytGYXsxuZSIymdC8GoMj1vUAZgJTgDHALSIyLDERxiZVygGpVZYGqVSmVClLqpSjKZooWt/XgG9wdG+Gs4F5xphyY0w1MJtQG2UyS5VyQGqVpUEqlSlVypIq5fgcvUbRyowxXwUQkcjVxRx9IXEvMKkNw2qxVCkHpFZZGqRSmVKlLKlSjqZojaJt2IRm8WtgEeoK296kSjkgtcrSIJXKlCplSYlyaKJoG7s4etLy7rSTG20aSZVyQGqVpUEqlSlVypIS5dCmp7YxF7hPRAoJDaNwOe3zDtlUKQekVlkapFKZUqUsKVEOrVG0AWPMbuAuYD6wCphljPkosVG1XKqUA1KrLA1SqUypUpZUKYfeR6GUUioqrVEopZSKShOFUkqpqDRRKKWUikoThVJKqag0USillIpKE4VSSqmo9IY7peIsPCPbt4FrCP3NuYE3gJ8aY+oTGZtSsdAahVLx9xRwMnCWMWYMMBEQQhMKKZX09IY7peJIRPoSmhu5yBhTGbG+O3CqMebVRMWmVKy06Ump+BoPrItMEgDGmBJAk4RqF7TpSan4CqB/Z6qd0//ASsXXEmCoiORErhSRHiLypohkJCgupWKmiUKpODLG7AH+CjwvIp0Aws9PAmXGmNpExqdULDRRKBV/XwfWA4tEZBWhWsZ64KsJjUqpGGmvJ6WUUlFpjUIppVRUmiiUUkpFpYlCKaVUVJoolFJKRaWJQimlVFSaKJRSSkWliUIppVRUmiiUUkpF9f8BMVNToNB8phYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ce_losses = [\n",
    "    get_train_and_validation_ce_loss(C, 3) for C in 10**np.linspace(-7, 5, 100)\n",
    "]\n",
    "plt.semilogx(10**np.linspace(-7, 5, 100), ce_losses)\n",
    "plt.legend(['train', 'validation'])\n",
    "plt.ylabel('Mean Cross Entropy Loss')\n",
    "plt.xlabel('C')\n",
    "plt.savefig(\"cross-entropy-loss-vs-c-degree-3.png\",\n",
    "            dpi=300,\n",
    "            bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEQCAYAAACugzM1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3xV9f348dc5d+VmELLYhM2HEcIGWYrKtrW14qizw1q/tlpbO2zV2tZa/VmtdmiHrdrWarVuK2JFkI0QZMn4sAmEhAxIQuZd5/fHvWAYSW4gN/fm5v18PO4j96x73x8S7vuezzQsy0IIIYRojBntAIQQQsQ2SRRCCCGaJIlCCCFEkyRRCCGEaJIkCiGEEE2yRzuAVuYCxgOFgD/KsQghRHthA7oD64D60w/GW6IYDyyPdhBCCNFOTQNWnL4z3hJFIcCxY9UEArE7PiQjI5mysqpoh9Eq4qUs8VIOkLLEqlgui2kapKUlQegz9HQRTRRKqeuA+wAH8KTW+qkGx0YBzzc4PQs4prXOUUplAy8AXQANXK+1Dudf2A8QCFgxnSiAmI+vJeKlLPFSDpCyxKp2UJazVtlHrDFbKdUTeAiYCowCblVKDTtxXGu9UWs9Sms9CpgMHANuCx1+Gnhaaz0EyAPuj1ScQgghmhbJXk8zgMVa66Na62rgVWB+I+f+GFiqtV6hlHIAF4bOh+Bdx1URjFMIIUQTIln11INT67sKgQmnn6SUSgVuBUaEdmUClVprX4PrerXkjTMyklscbFvLykqJdgitJl7KEi/lgPgoSyAQoLi4mKNHj+H3t/9OjMXF0Y4AbDYb6elpZGZmYprh3ydEMlGYQMMKOQMInOW8G4A3tdYn/hlPv45GrmtUWVlVTNcFZmWlUFJyPNphtIp4KUu8lAPipyxHjxZjt5ukpmZhs9kxDCPaIZ0Xu93E52vRR1mrsiwLv99HeXk5x44dJz29y8ljpmk0+QU7klVPhwj2yz2hG3D4LOd9Efh3g+1iIFUpZQttd2/kOiFEHPN46khLy8Rud7T7JBELDMPAbnfQuXMGHk9di66N5B3FIuBnSqksoBq4kmAV00lKKQMYC6w+sU9r7VVKLQeuAV4EbgLei2CcJ/mPHsSqqQTDAMMEw8AwTDBNMGyhn8Ftw7SHntvAtGHY7GAGH0YLbumEEI2xgv//WlahIJoR/DdtWY1LxBKF1rpAKXUvsARwAn/VWq9VSi0Afqq1ziPYJdajtT49vd0O/F0pdR+QD3w5UnGeYHnrqHntp9Aa63MYJtjsYHNg2Bxgd2LYnWB3YThcFCUl47HsGA43hisx+HAmYSSkYLhTMNydMBI7B68RQogoi+g4Cq31iwTvChrum9fgeTHBKqnTrzsATI9kbKcL2Jwsy/4/ArUVuOwGDpuB0w5OW+i5DRw2cJgnHgHshoXDtLCbFqYVAMsPfh8EfFh+L/i84Pdi+Tzg92B567F89XiPFeGvrcby1oGnpvHk5EzETErDSMnETM7E7JSJmdodM60HRnKm3LkI0Yaqqqp46KGf8fDDj4V1/o4d23jzzde4557237s/3kZmnzO/32JTiY2SY27qvX7qPf4GN2dn+yA3Qo8g0zBwOW0kNHi4XXbcTjtul53EBDtJSXaS3A56dO2E5fWRkuikU5KdJHsAw1ONVXscq64Kq7aCQE0FVvUxrOqjBKpK8RbuBG/tZ29vs2Om98aW0Qczsw+2rgMx03uGbiuFEK3t+PFKdu3SYZ8/ZMgw7rlnWPMntgOSKEKcDhs/uWHsyW3LsvD4AtR7/Xg8/mDy8AZCP/14QsnkxHa910+dp+HDR229j/IqD7X1PqrrvHi8Z69rNQ2D1GQn6Z1cpKckkJHanS6d+9O1l5suaYmkd3IFT6yvJlBeSKC8EH/5YQJl+Xj3rYMdHwWPu5KwdR2IvVcO9uyRmJ26nPX9hBAt9+STv6a0tIQf//j7HDiwj9TUzrhcLh566FEefvhBSkqKKS0tYdy4Cdxzz/1s2LCeZ5/9C3/4w1/49rdvZfjwHDZu3EB5+THuuusHTJo0JdpFCpskikYYhoHLYcPlsEFi67ymzx+gutaLw+3kYEE5lTVeKqrqqaj2UH68nqPH68k/cpwNu0rw+T+7i0lw2uiZlUTPzGT6dkuhX/fR9Bw0FbvNxLIsrOMl+It24S/S+Ao19fmbqF/1L8y0Htj7jcMxaDJm6hk1fEK0Kyu3FLJi81mnIjpvU3O7M2VE9ybPueuuH3DHHd/kzju/x1VXXc5//vN7unfvwQcfLGTQoMH88pf/D6/Xyw03XIXWO8643ufz8uc/P8eKFct45pk/SqIQZ2e3maQmu8jKSiHR1nh3v4BlcayynuJjNRQdq+VwSTUFpVWs18Us2xTsKeywmwzo0Ykh2WkM6ZNG/wGTSBgc/MMLVBzBl78J34ENeDa8g+eTtzG7DsQ55CLsAy8INrALIc5ZWlo63bv3AGDmzDls2/Ypr7zyIvv376OiooLa2pozrrnggskA9O8/gOPHK9s03vMliSIGmYZBRmoCGakJDO372X7LsiipqGN/YSV7CirRB4/x1op9vLliH26XndwBGYwamEnugAzcI2bhHDGLQPUxfLtX4925grqlf8NY9xqOnBk4h16M4UqKWhmFaKkpI5r/1t9WXC7XyeevvvpvPvpoMZdffgXz509g3749WGfpoOJ0Bq8xDOOsx2OZJIp2xDAMunR206WzmwlDuwJQVetF5x9j054yNu0u5eNtR3DaTcaoLCbndGNYn3ScI+fhyJ2Lv2Arnk3v4Vn7Kp5N7+Ea+0Ucwy4OjgkRQjTJZrOddSqRdes+5vLLv8SsWXPYsWMbu3btJBAItGiKjFgnnxDtXLLbwVjVhbGqC4GAxe6CCtZsO8LabUdYs/UImakJzBjbi2kje+DulYO9Vw7+0v3Uf/wK9av+hXfrh7gmX4e9d260iyJETEtPz6Br12786lc/P2X/1Vdfx2OPPcwLLzxHUlIyOTm5FBYepmfPFk1RF9OM9nYL1Iy+wD6Z6wm8vgAbd5eyKO8guw5VkOC0cfHonsy9oA/Jbkdw3pf8TdSt+TdWRRGOIdNxTboWw5HQoveJl3mF4qUcED9lKSo6QK9e/aI6P1JrivZcTw0VFR2gW7c+J7cbzPXUD9h/+vlyRxGnHHaT8UO6MH5IF/YVVvL+2nwWfpzPRxsLmD0hm5njeuPuM4qkXsPx5L2BZ9N7+A5vw33xrdi6Dox2+EKIGBI/lWiiUf26d+K2L+Tw869PYEh2Gm8u38dPnlnD2u1HwLTjmng17s/fA1aAmrcfxrNjabRDFkLEEEkUHUivrGTuuDKXe28cS2qSkz+9tZUnXtlE8bEa7N0VSVf+AlvPodQve466NS9jBWLjNlkIEV2SKDqgAT1Tuf/mcXx5xiB2F1TwwLPrWL75MDjcuOd8F8ewS/Bufo+6RX8IzlklhOjQJFF0UDbTZOa43vzylon0657Ccwt28Oe3t1LrCZAw9SZck67Dt/8T6hY9jeX3Nf+CQoi4JYmig0vvlMD3rx3NlRf1J29HCT97bh0FJVU4R8zCNeUGfAc2UPfhH7ECkiyE6KgkUQhM0+CySX2554YxeH0BfvXCerbsLcM5fEbozmI9dYv/gmVJm4UQHZEkCnHSwFDbRVaqmyf/s4lFeQdxjpiFc8LV+PauxZP3RrRDFKJdeOihn7FgwTuUlpbw/e/fedZzpk4d1+RrHD5cwMMP/wIIrm3xyCMPtnqc4ZJxFOIU6Z0SuOeGMTzzzjZeXLSL2nofn5s8F6uyCM+GdzDTe+EYMDHaYQrRLmRmZvHYY787p2uLigopKDgERH9tC0kU4gwJTju3X5HDs+/u4I3l+/D5Lb4w+UYCxwqp++hvmKldsWX2jXaYooPx7lyJVy+LyGs71IU4Bjc97fdPfvIDZs2aw/TplwLwta/dwB13fJe//OVp6uvrOH68ijvv/C7Tpk0/eU1h4WHuuOObvPrqOxw+fJgHHriX2tpahg/POXlOSUkxDz/8IFVVxyktLWHevM9zyy238dvfPsbhwwU8/vj/4+KLLz25tkV+/gEeffQhjh+vJCHBzV13fZ+hQ4fz0EM/IykpGa23U1pawle+cguXXXZ5q/z7SNWTOCubafL1y4YyNbc776zaz+sr8nHN+BZGQjK17/8Oq64q2iEK0aZmz57HokXvA3DwYD4ej4fXXnuZe+65n2ef/Rf33HMfzzzzx0avf/zxR5g37/M8//yLjBgx8uT+Dz54n5kzZ/OXvzzPP/7xMq+88hLl5eV85zvfR6mh3H33j055nQcfvJ+rrrqWv//939xxx/e4774f4fF4ACguPsLTT/+VRx75DU899dtWK7vcUYhGmabBV+YOwW4aLFhzgGS3g5mz7qDmzV9St+IfuGfcHu0QRQfiGDyl2W/9kTR58lSeeOJRamqqWbTofWbPnsvVV1/HqlXLWbJkEVu3bqG2trbR6z/5ZD0PPPAQALNmzT3Z5nDddTfyySd5vPjiP9m3bw8+n5e6urO/Tk1NDYcOHeKiiy4BICdnBJ06dSI//wAAEyZMxDAM+vcfQGVlRauVPaKJQil1HXAf4ACe1Fo/ddpxBfwZSAOKgGu11seUUjcDjwBHQqe+q7W+N5KxirMzDYMbZiuq63y8smQ3aSnDGTX2C3jyXse7ewxkzYx2iEK0CYfDwZQp01ixYhmLF3/Ar3/9W771rW8wZsxYRo8ey9ix4/n5z+9r9HrDME5OVmoYBqZpA+D3v3+Cw4cLmDlzDhdeOJ28vLWNrldxtp6HlsXJ6c8brnnRmiJW9aSU6gk8BEwFRgG3KqWGNThuAG8Dj2itRwIbgHtCh8cB39Najwo9JElEkWkY3PK5oQzulcrf3t3G/vTJmF36U7fyn/gqy6IdnhBtZvbsefz73y+QmtqZxMREDh48wNe/fhsXXDCF5cuXEmhi2pvx4yfy/vsLAFi6dDEeTz0AeXkfc911N3LJJTPIzz9ASUkxgUAAm81+xvoXSUnJ9OjRk6VLFwPw6adbOHq0jP79B0SoxEGRbKOYASzWWh/VWlcDrwLzGxwfA1RrrReGtn8FnLjjGA/crJTaopR6QSmVFsE4RRgcdhvfvjKXrM5ufv/GNmrG3Ag+LyXvPt3uVusS4lzl5o6iqqqKWbPm0qlTKp/73Be48caruf76+dTU1FBXV9do9dPdd/+IpUsXc/PNX2b16pUkJgZXmLzhhq/w4IM/5cYbr+a1115hyJBhHD5cQN++famqOs6DD95/yuv89KcP8p///JubbrqGJ554lIceehSHI7LLG0dsPQql1I+BJK31faHtW4AJWutbQ9vXADcDhcBoYDtwh9b6qFLqDeAxYBXBBJKttb4+jLfti6xHEVEl5bX84vl1ZKQm8KOxFfjW/IuE6d+Iat1xa2jPv5PTxUtZZD2KyIml9ShMoOGntQE0/FeyA9OBC7XWeUqpB4HfAF/RWl9x4iSl1KPAnpa8cajAMS0rKyXaIZyTrKwUvnf9WB7828f892h/vtB9IN68V+k27iJMlzva4Z2X9vo7OZt4KEtxcbDCw26Pn86ZsVIW0zRb9DcSyURxCJjWYLsbcLjBdhGwS2udF9p+CXhVKZUKfE1r/URovwG0aKIhuaOIrH5ZScydmM17aw4w6rLP0231Exz+4EVcE6+OdmjnrL3/ThqKl7KcqO+PlW/h5yuW7igCgcApfyMN7ijOKpLpbRFwqVIqSymVCFwJLGxwfBWQpZQ60aH488B6oAr4oVLqxPDfbwMyd0SMueLC/gzqlcoTH5Tj63MBni3vEygvinZYIq4YMr9YBAT/TVvWKypiiUJrXQDcCywBNgIvaq3XKqUWKKXGaa1rgSuAZ5RSW4FLgLu11n7gauCPSqntwFjgh5GKU5wbu83km5cPx2Ya/KMkB2xO6la/GO2wRBxxOhM4erQUn88rHSZagWVZ+HxeystLcToTWnRtxBqzo6Qv0pjdpjbuPcrvXtnI3bklZB96D/fcu7H3HhHtsFosnn4n8VKW4GdTHSUlZQQC/mbPj3WmaTbZfbZtYrDhdieTnJx6yliLaDZmiw5gxoRsPlyXzx+3w6+6plOf9wa2XjmtPuBHdDyGYZCV1QXDaN+dJE5ozwk8rKonpVRK6OdopdSNSqnIdtoV7YZhGNw8R+E3bCwLjCZQshf/wU3RDksI0YqaTRRKqV8Af1ZKZRNsjP4q0PjMV6LDyUx1c/X0Abx5uDseVzr1eW9KnbIQcSScO4p5wC0Eey29pLW+BBjZ9CWio7lodE+yu6WyoDqHQOl+/Ac2RjskIUQrCavqSWtdQ2hKjtAuV8QiEu2SaRh8ecYgllZmU+NIo379G3JXIUScCCdRlCmlniY4Ud8ipdQjnDpwTggABvXqzLih3XircjiBsnx8BzZEOyQhRCsIJ1HcRDAxXBa6s7AIztEkxBnmTx/ABm9/qsxOeDcvbP4CIUTMazZRaK2PAL8Nzcc0GtgGHI14ZKJdykx1M3NCXz6oGoy/aCf+kv3RDkkIcZ6k15NodfMuyGarORQPTjxb3o92OEKI8yS9nkSrS3DauXjiIFbVDcC752MC1ceiHZIQ4jxIrycREReP7sl6YwSWZeHduija4QghzoP0ehIR4XLamDQxh82e3tRtXYLlrY92SEKIc9SSXk/zpNeTaInpo3uylpGY3hq8u1ZGOxwhxDkKt9fTP4Gc0HKmfwvtE6JJLoeNERPGc8iXxvHNH0Y7HCHEOQqn19NsIA/4InA5sE4p9YVIBybiw/TRPdlgDcFZWSBdZYVop8KpenoQuEhrfYXW+nJgKvCziEYl4obTYSM150I8lo2KjdKoLUR7FE6icGqtt53Y0FpvBWyRC0nEm2njB7DJ2xf2r8Xy1kU7HCFEC4WTKGqVUuNObISe10QuJBFvOiU6qe51AQ7Lw/Htq6IdjhCihcJZ4e6HwH+VUrsI9ngaAlwV0ahE3Bk7ZRJFr/4X96bFdMq9JNrhCCFaIJxeT8uBYcCvgEeBoVrrpZEOTMSXHlnJHEgeSWrtIepLDkQ7HCFEC4Q7Mvuo1vo9rfV/tdZlSqkVkQ5MxJ9eE2fgs0wOrf5ftEMRQrRAOFVPZ5MbzklKqeuA+wAH8KTW+qnTjivgz0AaUARcq7U+FpqA8AWgC6CB67XWVecYq4gRalBvNnyUTfeiDViBAIYZ1vcUIUSURex/qlKqJ/AQwe60o4BblVLDGhw3gLeBR7TWI4ENwD2hw08DT2uthxAcw3F/pOIUbccwDOg3kWRqOLx1fbTDEUKEKZJf6WYAi0PVVtXAq8D8BsfHANVa6xOr2/wKeEop5QAuDJ0P8DzSeB431OSLqLUcHNu8LNqhCCHC1GjVk1Lqd40cMgBnGK/dAyhssF0ITGiwPRAoUkr9DRgNbAfuADKBSq21r8F1vcJ4v5MyMpJbcnpUZGWlRDuEVtOSsmRlpfBRp6F0rdxOYoJJUkpSBCNrmY76O4l1Upboa6qNoqyJYw+H8domwe60JxhA4LT3ng5cGFo970HgN8C9p13Hadc1q6ysikDg9JeIHVlZKZSUHI92GK3iXMqSMmwqCR9vZvW7Cxh56ZwIRdYyHf13EqukLG3DNI0mv2A3mii01j8/z/c+BExrsN2NU6cnLwJ2aa3zQtsvEaxuKgZSlVI2rbUf6I5Max5XeuWMpejjJAJ712BdMjvYdiGEiFmRbKNYBFyqlMpSSiUSXCFvYYPjq4AspdSJ1fI+D6zXWnuB5cA1of03Ae9FME7Rxkybjequo+gbyOdAflG0wxFCNCNiiUJrXUCwGmkJsBF4UWu9Vim1QCk1TmtdC1wBPKOU2gpcAtwduvx2gr2kthG8K7kvUnGK6Ogx7lLsRoADaz+KdihCiGY0O45CKWVv0LDcIlrrF4EXT9s3r8Hzjzm1gfvE/gME2y9EnErs0Y8CWzqdSjfj8V6N0yHzTAoRq8K5oziglHpIKdUn4tGIDsMwDOgzlv5mEVu27Y92OEKIJoSTKC4AfMAypdR/lVKXhQbLCXFeuo6cimlYHNmyOtqhCCGaEM6kgAe11g8A/YBngN8D+5RSP1BKuSIdoIhf9sxsqh1pZFZso7LaE+1whBCNCKsxWyk1FPg18CeCDdN3AH2BVyIWmYh7hmFg7zuWwfZC8jbvj3Y4QohGhLNm9gpgMVALjNdaf0lr/Q7BZDGtyYuFaEba8MnYDIuybR9HOxQhRCPCmT32aeA/ofENJ2mtA6FZXoU4Z2ZWP+qdqfSq0hwqqaJXVuxPvyJERxNO1dMrwDeUUm8ppV5XSn31xAGZ+lucL8MwcPYfzxDHYdZukgWNhIhF4SSK3wJXA28BC4CvKaV+GdGoRIeSNHgidiPA8V15WFbsztElREcVTtXTLGDYiaonpdQLwCZktLRoJWbXAXidnehftYc9hysZ2DM12iEJIRoI546ihFMTSgAoj0w4oiMyDBNn39EMdRwm79OCaIcjhDhNOHcUG4HlSqnnCQ68uwYoVUp9D0Br/ZvIhSc6ioT+Y7F2LuXo7k0EZg7FNGVMpxCxIpxE4Qa2AGND2/tDP0dw5roRQpwTW8+h+G1O+tftY+fBcob0SYt2SEKIkGYThdb6qwChuZ4cWuvdEY9KdDiGzYG91why9m1n8bYiSRRCxJBwBtwNDE0DvhFYr5Tao5QaEvnQREfj6jeGVLOGwl3b8flbtKihECKCwmnM/gPwqNY6TWudCvyS4CA8IVqVPXskFgYDrH3sOHAs2uEIIULCSRRdtdZ/P7GhtX4OyIpcSKKjMhKSMbsNItd1iI+3H4l2OEKIkHAShV0plX5iQymViTRiiwhx9h1Dd/MYB3bvk+onIWJEOIni98AapdSDSqlfEFzr+o+RDUt0VPY+owEYENjPjnypfhIiFoSTKJ4Dvgk4gUTgdq21JAoREWZqV4zOPRjhOsR6XRLtcIQQhDeOYp3WehSwJNLBCAHg6DOKAeULeXlnAYFZSgbfCRFl4dxRVCulekU8EiFCbNkjMQnQw5vPrkMyW4wQ0RbOHUUSwaVPDwInpxXXWuc2d6FS6jqCkwc6gCe11k+ddvwB4GvAicroZ7TWTzW2P4xYRRywdR0IzkRyXAXk6RJUtgy+EyKawkkU3zmXF1ZK9QQeIjj1Rz2wSim1RGu9rcFp44BrtdarT7u8sf2iAzBMG/ZeOYzY9ynv7SzmyzMGYRpS/SREtISTKG7SWn+94Q6l1KvA0maumwEs1lofbXDNfOAXDc4ZB/wkND3IMuD7Wuu6JvaLDsKePRL33rUk1xxm3+FKBsjU40JETaOJQin1R6AnME0p1XCAnQPoH8Zr9wAKG2wXAhMavH4ysAH4AbAbeB64Xyn18Nn2A/eG8Z4AZGTE/nKaWVkp0Q6h1USiLP6kyRz46K/kuArYdrCCC0ZFvplMfiexScoSfU3dUfwNyAFGAq812O8D1oTx2ianDswzCK5lAZxcRnXeiW2l1OPAs1rre8+2nxYkirKyKgKB2B0TmJWVQknJ8WiH0SoiVxYDs+sAxpQW8acNh/jcxN4YEax+kt9JbJKytA3TNJr8gt1ootBa5wF5SqlFWutD5/Deh4BpDba7AYdPbCilsoEZWutnQ7sMwNvY/nN4f9HO2XvnknXkdeorj3GwuIrsru3z25gQ7V04bRS9lVL/BNIJfmgDYfV6WgT8LFRtVQ1cCdza4Hgt8KhSagnBNS6+BbzRxH7RwdizR+LJe53hzkN8srNEEoUQURLOOIo/A+8Q7P10R4NHk7TWBQSri5YQnKL8Ra31WqXUAqXUOK11CcER3+8AmmASeryx/S0tmGj/zIxsjKQ0JnQq5pOdpdEOR4gOK5w7Ct+5LneqtX4RePG0ffMaPH+NU9s/mtwvOhbDMLD3HkmfnaspLK2g+FgNXdISox2WEB1OOHcUnyqlRkQ8EiHOwpadiz1QTz97idxVCBEl4dxR9Ce4st0Bgu0HQHgjs4U4X/aew8C0cUFqMWt2ljBnYna0QxKiwwknUYTdLVWI1mY4ErB1H8Kw0gL+VVBBRVU9qcmuaIclRIfS1IC7bK11vtb6jBHYSqk5kQ1LiM/Ye+eSVPASaWYVG3aVMn10z2iHJESH0lQbxZsnniilTm9Y/lVkwhHiTPbsYC1nsPeTrFEhRFtrKlE0HAZ7+pQdMkObaDNGajeMlCzGphxh+4Fj1NTJ+Esh2lJTicJq5PnZtoWIGMMwsGfnklW7HyPgY9OesmiHJESHEu4dhRBRZe89EiPgZWRKmVQ/CdHGmur1ZCql0ggmDFuD5wC2iEcmRAO2HkPA5mBKSil/2luGx+vH6ZA/QyHaQlN3FCOA0tBjBFDWYDsn8qEJ8RnD7sTWYyjZvn14vAG27jsa7ZCE6DCamj02nFHbQrQZe3YujoObyXZX88nOEkYPzmr+IiHEeZNkINoNe/ZIAC7pepSNu0vx+QPNXCGEaA2SKES7YaZkYab1QNkOUV3nY+fB8miHJESHIIlCtCu23iNJrNhLisPPeun9JESbaFGiUEo5QyvQCREV9uxcCPi5tMdxNuwsIWDJkB4hIq3ZRKGUukIp9XulVAqwE9iklPpO5EMT4ky2boPA6WaU+zDlVR72FlRGOyQh4l44dxQ/Bv5CcCnT1UA2cGMkgxKiMYZpx94rh7Tju7DbIE8XRzskIeJeOInC0FpvAWYA72mtj4d5nRARYc8eCbUVXNTbz3pdjCXVT0JEVDgf+AGl1NXAbOB/Sql5gPRLFFFj650LGExILaassp79RcejHZIQcS2cRHE3cCvwE611EcGFjO6MaFRCNMF0d8LM6ke32j3YTIO8HVL9JEQkNbvCndZ6BcFqJ5RSTuDLWuv8cF5cKXUdcB/gAJ7UWj912vEHgK8Bx0K7ntFaP6WUGgX8FegELANu01r7wiuS6AjsfUbiyXuTMdkzydPFzJ8+AMOQeSyFiISI9XpSSvUEHgKmAqOAW5VSw047bRxwrdZ6VOhxIpG8AHxbaz2Y4ESE3wi/SKIjsPcZDVhcmFFCSXkd+Ueqoh2SEHHrXHo99SG8Xk8zgMVa66Na62rgVWD+aeeMA36ilNqslPqDUrHfnK0AACAASURBVCpBKdUHcGut14TOeR64Koz3Ex2Imd4bIzmDbO9eTMOQ3k9CRNC59HqqDPO6HkBhg+1CoNeJDaVUMrAB+AEwBugM3N/cdUJAaDGjPqOgcBvDs5PJ2yG9n4SIlGbbKPis19Mc4Pst6PVkcupKeEbD67TWVcC8E9tKqceBZ4EFTV0XjoyM5JacHhVZWSnRDqHVRKssNbmTKdr6IXP71fHoR16qvBb9e3Y659eT30lsaq9lCX5xsTCMz75Xt9eyhJMo7gZ+BvxYa12klAq319MhYFqD7W7A4RMboalAZmitnw3tMgBv6LrujV0XjrKyKgKB2P12mZWVQklJfHTpjGZZrMRscCTQs2YnNrM/C1fu5aqLB57Ta8nvJDa1t7L4S/fj27MWf+kBAmX5YFkkXHwL9uxRTZYlUFOO/9BW/MV7MTN6Y++di5mcDoDl8xCoLCZw9CD+0nwCxw5hdu6BQ03Dlv5ZZYtlBU5JSi1hmkaTX7DD7vWklOqjlBqotZ4S5nsvAn6mlMoCqgm2cdza4Hgt8KhSagmwH/gW8IbW+oBSqk4pNUVrvZJge8h7Yb6n6EAMmwN77xH4CzYzvO9Y1m4/wpXTB2BK7yfRyizLwrdrFf6jhzASkjCcSZhJaZhpPTCSMwlUFOHJex3fvjwwbZhpvbD3GYW/9AC1C3+Lc/yVWDOvDZ635QN8Bz4Bw8RwuCAQIFBRFHwjmx38PuoBs3MPLF89VtVRTlaymHbMzt3wFmzDu+V9zKz+mEmdCVQUEagoxjlyLq7xV7Z6+ZtNFEqpQcCbBNsOTKVUKXCZ1npHU9dprQtCdx9LACfwV631WqXUAuCnWus8pdQ3gXdCx1cAj4cuvx54RinVCfgE+N25FU/EO3uf0fj2rmP6QC+/21vP3oJKBvZKjXZYoh2zfPVgOjDM4LfzQPUx6pY9i//gFjBtEPCfeoHNAQEf2F04x3wBZ+5sDGfiydeqW/osnnWvcmjfGrylBWCawV57dhf46iHgx66mYu+Vg5mRTeBYIf6Dm/Ad3oHpSsRM7YaZ2hUzvRdm5+4Ypp1A3XF8u1bh3bmKQEURZmo37NmjsA+cFJF/E6O5BkCl1ELgJa3130PbXwVu1FpfEpGIzk9fYJ9UPbWdaJfFqqui6p93YuTM5e6VWVyY24PrZw1u8etEuxytScpydoHaSupXvoCtywAcQ6ZhOBOxrAD+/E14ti0hUF6IVVsZ/PC2u7BlZGOm9cS7dy0EfLgmXo1j2CXg92HVVWFVleEvP0ygvBDD7sQxfAam+8w2Msuy8GxaQGDnUsz+F+AYdglmYudWKVNraVD11I9gDc8pwmmj6HoiSQBorZ9TSn2v1SIU4jwYCcnYug3CKtjEyAHXsG7HEa6dMRCbKdORic9YPg+17/+WQMlefHvXUr/+Dez9xuM/shOr4ghGUjq2boMxElMxElKwasoJlB7Au2cNtoxsEi76GmZqt+CL2Z0YyemQnB6czbgZhmHgGnUZWTOvbbcJPJxEYVdKpWutjwIopTI5tVeSEFFl7zOa+jX/ZuoEO3nay44D5Qzvlx7tsESMsKwAdR89Q6B4LwkzbsdMycKz9QN8u1djZmTjuuQ27P3HYZhnfhxaliUj/gkvUfweWKOUeplggrgWeCKiUQnRAvZ+Y6lf828GBfbgdqXw8bYjkigEAFbAj2fda/j2rsM18Roc/ccD4J7+DayLvg4YTSYCSRJB4SSK54BdBMdR2IDbtdaLIhqVEC1gpmRhZvYlcOATxgyaz/qdJdw4ezAOuy3aoYko8JcdxLcvD/+RXfiL94K3DsfQi3HkzjnlvHPtStoRhZMo1mmtRxHsvSRETLL3H4dn7atMmepi5ac+Nu0uY9yQLtEOS7QRq74ar16Od9dKAmUHwTAw07NxDJqCrbvC3m+s3B2ch3ASRbVSqpfW+lDEoxHiHDn6BRNFP99u0lKSWbGlUBJFB2B5avBs+R+eLe+DpxYzqx+uyTdgHzDhrD2QxLkJJ1EkAfuUUgeBk1N0aq1zIxaVEC1kpnbDTO+Ff/96Judcw4I1ByivqqdzsivaoYkI8erl1K1+CTw12PuOwTnmC9gy+0Q7rLgUTqJodkpxIWKBvd94POvfZOq4RN5dDau3FjF3onxwxBvL56F+1Qt4dyzD1l3hmnSdJIgIa7I1RymVAWzXWi/VWi8l2Ji9LfRciJhi7zcOsEgr387Anqms3FIkM8rGmUBlMTVvPYR3xzKcoz6H+7IfSZJoA40mCqXUcGAH0HBupy8Bm5VSKtKBCdFSZloPzNRu+PauY8qIbhwurZb1tOOEZVl4d66k+rWfEjhegnv2d3BNmH9ymg0RWU39Kz8CfEdr/caJHVrrbxNcyOjRSAcmREsZhoG9/3j8hZpxfRNx2E1WbCls/kIR06z6auo+/CN1Hz2DLbMPSfMfDK1wKNpKU4kiW2v94uk7tdbPAwMiFpEQ58E+YAJYARwF6xk7OIuPtx7B6/M3f6GISf7a49T89xF8+9bjHH8l7st+hJmcEe2wOpymEkVT/7s8rR2IEK3Blt4bM7033l2rmZrbnZp6H3k7SqIdljgHVn01hS/+gkB5Ie6538U1+vNS1RQlTf2rH1FKjTp9p1JqNMH1JYSISfaBkwgU72Fwmpeu6Yks3iBDgNoby1NDzYLH8JTk4555B/ZeOdEOqUNrqnvsg8BbSqmfA6sIJpVJwE+BW9ogNiHOiWPgRDxr/4N/zxouHj2Gf3+4iwNFx+nTrX0uQ9nRWPXV1Lz3OIHSfLrO/wE1aUOiHVKH1+gdhdZ6FcHV5W4A1hJMFvOB67XWH7RNeEK0nJmcga27wrtrNVNyuuK0myyRu4p2IVBbSc1//x+B0nwSZn6LpMHjox2SoJkBd1rrZUAsLlAkRJPsgyZRv+w53FUFXDC8K2u2HuHqiweSmOCIdmiiEYGacmrffZRAZQnu2Xdi7y2TP8QKaRkSccnRbxyYdry7VnPx6F54fAFWbimKdliiEVZdFbX/fZTA8TLcc78nSSLGSKIQcclwJWHPHolvzxqyuyQyoEcnFm8okJHaMcjy1lOz8AkCx4txz7kLe4+h0Q5JnEYShYhb9kGTsWor8R/awsVjenLkaA1b9x2NdliiASvgo3bRUwRK9pJwyf9JkohR4UwKiFKqD5AOnJzQXWv9SaSCEqI12PuMxHB3wrt9KeMvvYNXP9rDex/nk9NfBmzFAivgo27JM/gPbsY17Ss4+o2NdkiiEc0mCqXUL4DvA8V8tla2BfQP49rrgPsAB/Ck1vqpRs67DPiD1rpfaPsi4HXgYOiUDVrrrzb3fkI0ZJh2HIOn4tm8EFd9BbPGZ/PKkt3sK6ykX3dZqyCaLG89tYuewn9wM84JV+McOj3aIYkmhHNHcSMwUGt9uCUvrJTqCTwEjAXqgVVKqSVa622nndcVeIwGdyvAOOAxrfXDLXlPIU7nGHIRnk0L8OrlXDRqHu+s2s97aw5w+xUjoh1ahxWoO07te08QKN2Ha9pXJEm0A+G0URxsaZIImQEs1lof1VpXA68SHIdxur8CPz9t33hgllJqs1LqbaVU73N4fyEwU7ti6zEU746lJDhNLhnTk/W6hCPHaqIdWodk1VdT+84jBI4eJGHmtyVJtBPh3FF8qJR6FHgLqD2xM4w2ih5Aw6k7C4EJDU9QSt0JfAKsOe3acuAVrfXrSqnbgH9z6nTnTcrISA731KjJyoqfUcKxXpaqiXMpfuM3JB/fyzWzhvK/dQdZurmIb80fecp5sV6OlojFslg+L4UvPUqg8gjdr70Pd9/w7upisSznqr2WJZxE8ZXQz6sa7AunjcLkszYNCFYtBU5sKKVygCuBS4FeDS/UWt/W4PmflFKPKKVStdYVYcRLWVkVgUDsdoPMykqhpCQ+1kloD2Wx0odiJKRQ+vFC3DMHMjmnG4vW5jN7bE9SQ0ultodyhCsWy2JZAeo+/BO+/G0kXHIbVUl9qQojxlgsy7mK5bKYptHkF+xmE8WJBuZzcAiY1mC7G9CwCusqoDuQBziBHkqp5cBFBNe8eERr3XAGW985xiE6OMPmwD54Ct4tHxCoKWfOxGyWbypkwZp8vjxjULTDi3uWZVH/8Sv49q7FOeFqHAMviHZIooXC6fWUSbBBO5ngXYGNYOP29c1cugj4mVIqi+Bss1cCt544qLV+AHgg9B59gY+01tNC21cAu4BXlFI3AR+H2jmEOCfOIdPxbl6Id9sSuo67gikjurFkwyFmje9NRmpCtMOLW5Zl4Vn7H7ybF+IYdinOkXOjHZI4B+E0Zr9CsGH660Bv4GYaVCE1RmtdANwLLAE2Ai9qrdcqpRYopcY1c/nNwF1Kqa3AV5HZasV5Mjt3w5Y9Cu+2xVg+D5dPCd4ov7NqX5Qji1+WZVG/+kU8mxbgGHYJrinXYxhG8xeKmBNOG0UfrfUApdTTwJ+BnwFvhvPioRXyXjxt37yznLcf6NtgeyswOZz3ECJcztw51P73Ebw7V5Ix7GKmj+7J4vUFzJnYp902MsYqy7KoX/lPvNsW48iZhWvSlyVJtGPh3FGcmEltF5ATulOQKThFu2PrrjAz++LZ8j6WFeBzk/risJu8uXxvtEOLO56814NJIneuJIk4EE6iKFZK/QBYB3xNKfV5IDGyYQnR+gzDwJk7B6uiCH/+ZjolOZk5vjdrtxez51B5tMOLG55tS/BseAfHkAtxTbxakkQcCCdRfBOo11qvINhD6RfAjyIalRARYu8/DiMpHc/mhQDMmdCbZLeDZ976VGaWbQW+/RuoX/kPbL1zcU29WZJEnGg2UWiti4FnlFIjCHZbnaK1fiPikQkRAYZpx5kzE3/hDvwl+0hMcHDlRf3ZureMj7cdiXZ47Zq/eC+1H/4RM7Mv7hm3Y5i2aIckWkmziUIpdQGwB3iX4Gjrg0opaWgW7ZZj6HRwJVG/PtgnY1puDwb27szLS3ZTWy/Ddc5FoLKY2oVPYCSm4p59F4ZDuhzHk3Cqnn5NsHtsmdb6EMExFb+NaFRCRJDhdOPMnYM/fxP+4j2YpsFtV4ygosrDf1ftj3Z47Y5VV0Xte7/BsgIkzv0eZmJqtEMSrSycRJHYcMZXrfUCwlzHQohY5cyZiZGQQn1esBZV9Ulnam53/rfuIIVlMrYzXJbPQ+3/fkfgeCnuWXdidu4e7ZBEBISTKLxKqTRC8zYppVRkQxIi8gxHAs6R8/Af+hRf0U4A5l80AJfDxnPv7YjpucJihWUFqPvoGfxFO0mYfgv27vLREK/CSRS/BJYCvZRSLwGrQvuEaNccwy/BcHfCE7qr6JTk5LqZg9h9qIIP8g42c3XHFhx1/RK+vetwXXCNzN8U58Lp9fRf4EsE52VaCUzVWr8W6cCEiDTD7sI56nP4D2+ndt9mACYN78aogZm8vmyvVEE1wbt5Id5PP8CRMwvHiDnRDkdEWKOJQimVfuIBHAVeJjgdx5HQPiHaPcfQ6RgpmZQteg4r4McwDG6ao3DaTZ59d7tUQZ2Fd9cq6j9+GXv/CbgmXStjJTqApu4oSoGSBo/S034K0e4ZdieuidfgKc7Hu2MpAJ2TXVw/czB7DleyYM2BKEcYW3z5m6j76G/YegwlYfotGEY4tdeivWuq99I/CE7M9xbw3OlrXQsRL+z9xpHQJ4e6da/h6D8BIyGZicO6snF3KW8u38fg3p0Z3LtztMOMOl/RLmo/eAozoxfuWXdi2J3RDkm0kUa/DmitvwKMAjYBv1VKrVZK3a6Ukv8xIq4YhkHmrK+Bp4b69W+c3HfznCFkdU7gT299SmW1J8pRRpf/aEFwQF1yOu65d2M43dEOSbShJu8btdY1WusXtNYzCa5IlwosUUq93CbRCdFGnF364Bh6Md5tS/AfDfZ4crvs/N8Xc6iq9fHMO1s7bHtFoKac2oW/wbA7SZx3N6a7U7RDEm2sJRWMWaFHJiB3FSLuuMZ9CcOZSN2y57ECwbW5srumcP3MQWzdf4y3V3a8RY4sbx21C5/AqqvCPecuzJSsaIckoqDJEdZKqd7ADQSn7fAD/wQmaq0PN3WdEO2RkZCMa8oN1C3+E95P38eZG1y288KRPdhdUMHbK/fTLSORC4Z1i3KkbcMK+Kn98I8EyvJxz74LW2bfaIckoqTRRKGUWgIogt1ir9dab2izqISIEvuAidj3fEz9utexZ4/G7Nwt2GV29hBKjtXy7Ls7yEp1M6BnfM9ndGKFOn/+JlxTb8KePTLaIYkoaqrq6SKCVUy3AEuVUpWhx3GlVGXbhCdE2zIMA9e0m8HupG7p305WQTnsJt/60gjSU1z8/rXNlJbXRjnSyPJ88jbe7R/hHHUZzmGXRDscEWVNJYp+wFAgBxjR4HFiW4i4ZCZ2JmHSdfiP7ML76f9O7k9JdPKdq3Lx+S2e+M8mKmvisyeUZ/tHeNa/gX3wFJzj50c7HBEDGq160lqf90gjpdR1wH0E19h+Umv9VCPnXQb8QWvdL7TdGfgX0J/g4L6rtdZFZ7tWiEiwD5qMff966tf+B1u3wdi69Aege0YSd87P5Tcvb+Q3L2/kh18eTWJC/Cwh793/CfUr/o6tdy4JF35VRl0LoGW9nlpEKdUTeAiYSnA8xq1KqWFnOa8r8BjQ8C/yl8ByrfVQ4Blk/QvRxgzDIOHCr2Ekdqb2w6ex6j+b92lw785860sjKCip5sn/bKbe449ipK3HV6ip+/BpzMx+uGd8C8OU1QREUCTH388AFmutj2qtq4FXgbPdx/4V+Plp+y4jeEcB8BIwVykVP1/bRLtgJCTjvvT/sKqOUbf02VPW1B7RP4NvXj6cPYcr+N1r7T9Z+MvyqV34JGZKFu6538VwuKIdkoghkUwUPYDCBtuFQK+GJyil7gQ+AdY0dq3W2gdUEhzDIUSbsnUdiGvifHz715/SXgEwbkgXbrlsGDvyj/H4KxupqWufy6gGKoupXfA4htONe973MRNSoh2SiDGRvLc0CS12FGIAgRMbSqkc4ErgUk5LIJxaDXXGtc3JyEhuUaDRkJUVP/8Z46UsjZXDuuQqjhzdS82al0nrO5DE/qNOHrv84hQyMpJ47IU8nnh1Ez//xiRSk6P/bTzc34mvsozD7z2GgZ8e1/8cZ1bvCEfWcvHy9wXttyyRTBSHgGkNtrsBDQfqXQV0B/IAJ9BDKbVcaz0NKAidf0gpZQdSgLJw37isrCqmp1vIykqhpOR4tMNoFfFSlubKYU7+GmZpEUWvPUbiF+/H1rnHyWODu6fw7S+N4A+vf8oPfreMu64aSVbn6M2FFO7vJFBTQc07D2PVVJJ42Q+poDPE2O8yXv6+ILbLYppGk1+wI1n1tAi4VCmVpZRKJHj3sPDEQa31A1rrwVrrUcA84HAoSQAsAG4KPb+GYMO2N4KxCtEkw+nGPfs7GDYHtQt/i1VXdcrx3AGZ3H3NSCqqPPzyH3nsKaiIUqThseqqqF3wa6zqo7jnfu9kry4hziZiiUJrXQDcCywBNgIvaq3XKqUWKKXGNXP5/cAFSqmtwO3AtyIVpxDhMlMycc+8A6uqjNr//Q7LV3/KcZWdxr03jSXBaePRlzawdvuRKEXaNKu+mpoFjxGoKMI96zvYuw2OdkgixhkNe3LEgb7APql6ajvxUpaWlMO752PqPvwTtuxc3LPuOKMb6fEaD79/fQu7D1Uwa3xv5k8fgN3Wdgv8NFUWy1NLzYJfEyg9gHvWHdizR531vFgRL39fENtlaVD11A/Yf8bxtg5IiPbOMWAirqk34Q+t9mZZp/azSEl08oNrR3PpmF78b91BHn1pA8eO1zfyam3H8tZR897jBEoO4J7x7ZhPEiJ2SKIQ4hw4h12Mc/x8fLtXU7/in2ckC4fd5PpZg/nm5cM5eKSKB55dS96O4ihFC5anhtr3fkOgeC8Jl96Gve/oqMUi2h8ZeinEOXKOugw8NXg2LYCAH9e0r2CYp373mjisK9ldk/nLO9t4+s1PmTisK9fPHEyyu+3Gj1p1VcE2ibKDJFz6fzj6j2+z9xbxQRKFEOfIMAycE64CmwPPJ29h+T0kTP8Ghmk75bzuGUnce+NYFqw5wDsr97PjwDGuuXQgE4d2jfhcSoGacmrffYxAZRHu2bHfJiFikyQKIc6DYRi4xl0Bdgeeta9S560n4dLbMOynDrqz20wun9KPkQMyeX7hDv7y9jaWbTzM9bMUPTOTIhKbv/wwte89gVVbiXvO97D3PGOqNSHCIm0UQrQC16jP4ZpyA74DG6l55xECNeVnPa9PtxTuv2kcN85WHCyu4oG/reUfC3e0emO37/B2at78JfjqSfzcDyVJiPMidxRCtBLn8BmYSRnULv4jNW8+iHvOXdjSz5wSwzQNLh7dk3Eqi7dX7OejjQWs/LSIS8f2Ys6EbDolOc8rjsqNH1K74M+YnbvinvNdWedanDcZRxEFsdyfuqXipSytWQ5/6X5qFz6J5aklYepNOAZPafL8kvJa3ly+jzVbi7DbTabmdmfOhOwWTwNi+TzUr3wBr16Gredw3DNux3BFplqrrcTL3xfEdlmaG0chiSIKYvkPpqXipSytXY5A9THqFv8Jf6HGPmgyCVNuxHA2/cFfWFbN+2vzWbmliIBlMXpQFheP6cmwPmnNNnoHKoqoXfQUgbKDdJ78JXzDLjujUb09ipe/L4jtsjSXKKTqSYgIMJPScF/2Izwb3sbzyVtUH9lNwuQbsGfnNnpN94wkvjJ3KF+Y2p8P1x9i2abDfLKzhK7piUzL7c6k4d1ISzm1kdwK+PBu+YD69W+AzYF7zndJHzs1Zj+QRPskdxRREMvfLFoqXsoSyXL4CjX1y54jUFGEvc9oXJO+jNmpS7PXeX0B8nQxSzYUsPtQBYYBw/qmM2FoF8YMziKh4gB1K/5OoOwgtuxRJEy9ETM5I25+JxA/f18Q22WROwohoszeXWGb/0s8W/6H55O38L3yY+wDLsCZOxtbRnaj1znsJpOGd2PS8G4cOVbDqi1FrN5ayIr/LcG9/FOUoxCPoxO2qbeRPHSirG8tIkYShRBtwLDZcY2ah2PQJDwb38Wrl+PbtRJbj6HY+4zC1jMHM63HWT/sLStApreIeUlbmNVlPYGyfOptSXzoncD7RwZQ/3YN2R+vY0T/DIb1TSe1c2IUSijimVQ9RUEs34K2VLyUpa3LYdVX49n+EV69HKuiCADD3QkjOQMjIQXDlYhVV4VVU06gqgw8tYCBmdUPh5oW7Ellc3C4rIZNu0vZtLuUPQWVBCwLp8PGgB6dGNy7M4N7pdK/RyouZ/ts2I6Xvy+I7bJI1ZMQMchwJeEadRmuUZcRqCrDd+hT/EW7sWrLsWorCJQXYiQkY6ZkYes2GFu3Qdh65ZyxnnXPzCR6ZiYx74I+1Nb70AfL2X+kio26mLdX7MMCDAN6ZibTv0cKfbt3ok/XFHplJeGwt8/kIdqeJAohosxMzsA55CIYctF5vY7bZWfUwExmTupHSclxauq87C6oYE9BJfsKK1mvS1i2qTD4noZBt4xEemQm0SP0s2taIl3S3Lhd8rEgTiV/EULEqcQEB7kDMskdkAmAZVmUVNSRX3Sc/OLjHCquJr/oOOt3FNOworZTooPMzm4yUxPI6JRAWoqLtJTgz05JDlKTnHI30sFIohCigzAMgy6d3XTp7GbckM+653q8fo4cq+XI0RqKy2spPlZDWUUdB4qO88nOEnz+M9v73C4byW4HyW4nKYkOEhPsJLrsJCY4SHTZcbtsuF12XA5b8OG04XTYcDlMnA4bTruJw25iM2W6ufZAEoUQHZzTYaN3l2R6d0k+45hlWRyv9XKssp5jVfVUVnuCjxoPVbVeqmq8VFR7KCyrpqbOR029j5b0j7GZBnabid124qeJzWZgMw1spkmCy0YgYIW2DUzTwDQMDMPAMAg9DzbGGoaBQbBNBjh5TnDTwDQAAwwa6UZsfPajqZ7GjZXv1GvOfAG320FtrbeJf43zN2l4V1R2Wqu/riQKIUSjDMOgU6KTTolO+pDS7PmWZVHn8VNb76O23ked14/HG6De68fj9Yd+BvD6Anh8fry+4HOfP4DPb+HzB/AHLPyhnza7jdo6L35/gIAFPm9wv2VZWBYErNOfE6xGsyysUDwnPtiDx87+Kd/ww7+xnqAWn338n96NueE1jeVJm2ngj3BvzN5dkiVRCCFim2EYuF32VmsQj+UupS3VnssS0UShlLoOuA9wAE9qrZ867fgVwM8BG7CO/9/evYVYVYZhHP+PkmGRSTUxahdC0ENdhOQhosONYgRBhRZklF3YFBUdMShBvFEkgyjSibDoQrMLIyqtCFEiETIvhFB4g8LwMHmRHchCTeti7QVrbLfcM86atdfH87uRvdaG/T7sWb77W4fvg/6IOClpMbAaONp669aIWFZlrWZm1l5ljULSNGAlMBM4AeyStCMi9rf2Xwy8AdwQEUclvQ88DLwFzAKei4hNVdVnZmadqfKWg3nA9og4FhHHgc3Awnxna9v0VpO4CLgS+KW1ezawWNK3kjZIGv2TbmZm1pEqTz1NBQYLrweBOcU3RMQpSXcAG4DDwBeF974C7AJWkY08Huj0g1uPone13t5zXxhsilSypJIDnKVbNTVLlY1iHENvAOgBzpz9poj4DLhc0ipgAFgUEffk+yW9DHw/nA/2XE9jJ5UsqeQAZ+lW3ZylMNdT+/0VfvYhYErhdR9wJH8h6TJJ8wv7NwLXS7pU0rOF7T3A3xXWaWZmJapsFNuAuZJ6W9cgFgCfF/b3ABsk5RPy3wvsBP4AXpB0Y2v7k8CHFdZpZmYlKjv1FBGHJS0DdgATgPURsVvSp8DyiNgjqR/YIukfYD/wWESclnQfMCBpIvAd8FCHHzsesmFUt2tCjZ1KJUsqOcBZulW3ZinU1XYSr9TWo7gF+KruIszMGupWsjM7Q6TWd3/uQwAAA7BJREFUKC4ku7V2EDhdcy1mZk0xnuya8jdkz70NkVqjMDOzUeY5fs3MrJQbhZmZlXKjMDOzUm4UZmZWyo3CzMxKuVGYmVkpNwozMyvlRmFmZqW8ZnYXkTSdbOnYX4EfImJdvRWNnKTxZBNDLo2IPXXXM1KSbgOWkE1i+WVErK+5pGGTdAXwGnCcbFnhj2ouacRS+D5yTTpGPKLoLs8DPwKTyBZtarKXKEwr32CTgUeBxcBdNdcyUk+RrVnfD/TXXcx5SuH7yDXmGPGIokaSlgK3FzadAd4EDpCtz3F3DWUNW5scA8A+/mcmym7WJst8sl+vq8l+lTdRH9n6MI0XER+3fok3+ftA0gIadIy4UdQoItYAa/LXkt4lW4/jT+BkTWUNW5scm4DfgFnA1cCDNZU2bG2yTAZeBdZ2++mBEgfJJnwbPNcbu10i3wfAQhp0jLhRdJf8P6ljwNs11zJiEXE/gKQVwJZ6qzlvrwNXAc9IOhgRL9Zd0AisB9ZIOkU2Ym2yFL6Pxh0jnj22ApLyawx3RsSB1rZFZBeqLyA7X7y2vgo7k0oOSCtLLqVMqWRJJcfZfDF7lLWWcN0JXFPYNg1YSbaw0gygX9J19VTYmVRyQFpZcillSiVLKjnacaMYfY8ATzD0boZ5wPaIOBYRx4HNZOcou1kqOSCtLLmUMqWSJZUc/+FrFKMsIpYASCpunsrQC4mDwJwxLGvYUskBaWXJpZQplSyp5GjHI4qxMQ4oXgzqIbsVtmlSyQFpZcmllCmVLEnkcKMYG4fIbk/M9dGQB23OkkoOSCtLLqVMqWRJIodPPY2NbcAKSb1k0ygsoJlPyKaSA9LKkkspUypZksjhEcUYiIjDwDJgB7AXeC8idtdb1fClkgPSypJLKVMqWVLJ4ecozMyslEcUZmZWyo3CzMxKuVGYmVkpNwozMyvlRmFmZqXcKMzMrJQfuDOrWGtFtqeBRWTH3ATgE2B5RJyoszazTnhEYVa9AeAmYG5EzABmAyJbUMis6/mBO7MKSZpOtjbylIj4vbC9D7g5Ij6oqzazTvnUk1m1ZgL7ik0CICJ+AtwkrBF86smsWmfwcWYN5z9gs2p9DVwr6ZLiRknTJG2VNLGmusw65kZhVqGIOAJsBN6RNAmg9e864OeI+KvO+sw64UZhVr3Hgf3ALkl7yUYZ+4EltVZl1iHf9WRmZqU8ojAzs1JuFGZmVsqNwszMSrlRmJlZKTcKMzMr5UZhZmal3CjMzKyUG4WZmZX6F6ZKOSc7TjoQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ce_losses = [\n",
    "    get_train_and_validation_ce_loss(C, 4) for C in 10**np.linspace(-7, 5, 100)\n",
    "]\n",
    "plt.semilogx(10**np.linspace(-7, 5, 100), ce_losses)\n",
    "plt.legend(['train', 'validation'])\n",
    "plt.ylabel('Mean Cross Entropy Loss')\n",
    "plt.xlabel('C')\n",
    "plt.savefig(\"cross-entropy-loss-vs-c-degree-4.png\",\n",
    "            dpi=300,\n",
    "            bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Intercept Regularization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Throughout the code above, we've ben setting `fit_intercept = False`. The reason for this is that we are using the `PolynomialFeatures` module, which automatically creates a feature that is always equal to 1."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For example, if we pick degree = 2 and use the features \"fare\", \"age\", \"pclass\", and \"sex\", we'll get the following features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['1',\n",
       " 'fare',\n",
       " 'age',\n",
       " 'pclass',\n",
       " 'sex',\n",
       " 'fare^2',\n",
       " 'fare age',\n",
       " 'fare pclass',\n",
       " 'fare sex',\n",
       " 'age^2',\n",
       " 'age pclass',\n",
       " 'age sex',\n",
       " 'pclass^2',\n",
       " 'pclass sex',\n",
       " 'sex^2']"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pf = PolynomialFeatures(degree=2)\n",
    "pf.fit(p3_train[[\"fare\", \"age\", \"pclass\", \"sex\"]])\n",
    "feature_names = pf.get_feature_names([\"fare\", \"age\", \"pclass\", \"sex\"])\n",
    "feature_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>fare</th>\n",
       "      <th>age</th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>fare^2</th>\n",
       "      <th>fare age</th>\n",
       "      <th>fare pclass</th>\n",
       "      <th>fare sex</th>\n",
       "      <th>age^2</th>\n",
       "      <th>age pclass</th>\n",
       "      <th>age sex</th>\n",
       "      <th>pclass^2</th>\n",
       "      <th>pclass sex</th>\n",
       "      <th>sex^2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>22.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>52.562500</td>\n",
       "      <td>159.5000</td>\n",
       "      <td>21.7500</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>484.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5081.308859</td>\n",
       "      <td>2708.7654</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>1444.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>26.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>62.805625</td>\n",
       "      <td>206.0500</td>\n",
       "      <td>23.7750</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>676.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2819.610000</td>\n",
       "      <td>1858.5000</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>1225.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>35.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>64.802500</td>\n",
       "      <td>281.7500</td>\n",
       "      <td>24.1500</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>1225.0</td>\n",
       "      <td>105.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>595</td>\n",
       "      <td>1.0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>19.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2819.610000</td>\n",
       "      <td>1008.9000</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>361.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>596</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.7500</td>\n",
       "      <td>31.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>60.062500</td>\n",
       "      <td>240.2500</td>\n",
       "      <td>23.2500</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>961.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>597</td>\n",
       "      <td>1.0</td>\n",
       "      <td>23.0000</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>529.000000</td>\n",
       "      <td>92.0000</td>\n",
       "      <td>46.0000</td>\n",
       "      <td>23.0000</td>\n",
       "      <td>16.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>598</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.4750</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>155.625625</td>\n",
       "      <td>74.8500</td>\n",
       "      <td>37.4250</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>36.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>599</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.5000</td>\n",
       "      <td>33.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>90.250000</td>\n",
       "      <td>313.5000</td>\n",
       "      <td>28.5000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>1089.0</td>\n",
       "      <td>99.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>600 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       1     fare   age  pclass  sex       fare^2   fare age  fare pclass  \\\n",
       "0    1.0   7.2500  22.0     3.0  0.0    52.562500   159.5000      21.7500   \n",
       "1    1.0  71.2833  38.0     1.0  1.0  5081.308859  2708.7654      71.2833   \n",
       "2    1.0   7.9250  26.0     3.0  1.0    62.805625   206.0500      23.7750   \n",
       "3    1.0  53.1000  35.0     1.0  1.0  2819.610000  1858.5000      53.1000   \n",
       "4    1.0   8.0500  35.0     3.0  0.0    64.802500   281.7500      24.1500   \n",
       "..   ...      ...   ...     ...  ...          ...        ...          ...   \n",
       "595  1.0  53.1000  19.0     1.0  0.0  2819.610000  1008.9000      53.1000   \n",
       "596  1.0   7.7500  31.0     3.0  0.0    60.062500   240.2500      23.2500   \n",
       "597  1.0  23.0000   4.0     2.0  1.0   529.000000    92.0000      46.0000   \n",
       "598  1.0  12.4750   6.0     3.0  0.0   155.625625    74.8500      37.4250   \n",
       "599  1.0   9.5000  33.0     3.0  0.0    90.250000   313.5000      28.5000   \n",
       "\n",
       "     fare sex   age^2  age pclass  age sex  pclass^2  pclass sex  sex^2  \n",
       "0      0.0000   484.0        66.0      0.0       9.0         0.0    0.0  \n",
       "1     71.2833  1444.0        38.0     38.0       1.0         1.0    1.0  \n",
       "2      7.9250   676.0        78.0     26.0       9.0         3.0    1.0  \n",
       "3     53.1000  1225.0        35.0     35.0       1.0         1.0    1.0  \n",
       "4      0.0000  1225.0       105.0      0.0       9.0         0.0    0.0  \n",
       "..        ...     ...         ...      ...       ...         ...    ...  \n",
       "595    0.0000   361.0        19.0      0.0       1.0         0.0    0.0  \n",
       "596    0.0000   961.0        93.0      0.0       9.0         0.0    0.0  \n",
       "597   23.0000    16.0         8.0      4.0       4.0         2.0    1.0  \n",
       "598    0.0000    36.0        18.0      0.0       9.0         0.0    0.0  \n",
       "599    0.0000  1089.0        99.0      0.0       9.0         0.0    0.0  \n",
       "\n",
       "[600 rows x 15 columns]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(pf.transform(p3_train[[\"fare\", \"age\", \"pclass\", \"sex\"]]),\n",
    "             columns=feature_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If we set `fit_intercept = False`, then our first coefficient is our intercept."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "unregularized_model_intercept_false = Pipeline([\n",
    "    ('scale', StandardScaler()), ('poly', PolynomialFeatures(degree=1)),\n",
    "    ('model',\n",
    "     LogisticRegression(fit_intercept=False, penalty='none', solver='lbfgs'))\n",
    "])\n",
    "unregularized_model_intercept_false.fit(\n",
    "    p3_train[[\"fare\", \"age\", \"pclass\", \"sex\"]], p3_train[\"survived\"])\n",
    ";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.5120583 , -0.00927245, -0.47776382, -1.06434141,  1.19627839]])"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unregularized_model_intercept_false.named_steps['model'].coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.])"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unregularized_model_intercept_false.named_steps['model'].intercept_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The model also has a variable called `intercept_`, but it's equal to zero. This is because we set `fit_intercept = False`. Note that even though the `intercept_` is zero, we actually do have an intercept of 0.51 because of the bias feature we created that is always equal to 1."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Thus, our model makes a prediction equal to $\\sigma(0 - 0.51 \\times 1 - 0.009 \\times \\text{fare} - 0.478 \\times \\text{age} - 1.06 \\times \\text{pclass} + 1.196 \\times \\text{sex}$."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's see what happens when we set `fit_intercept = True`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "unregularized_model_intercept_true = Pipeline([\n",
    "    ('scale', StandardScaler()), ('poly', PolynomialFeatures(degree=1)),\n",
    "    ('model',\n",
    "     LogisticRegression(fit_intercept=True, penalty='none', solver='lbfgs'))\n",
    "])\n",
    "unregularized_model_intercept_true.fit(\n",
    "    p3_train[[\"fare\", \"age\", \"pclass\", \"sex\"]], p3_train[\"survived\"])\n",
    ";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.25602854, -0.00927195, -0.47776545, -1.06434099,  1.19627759]])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unregularized_model_intercept_true.named_steps[\"model\"].coef_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This time, we see that our model is exactly the same, except that the weight of our bias feature is only -0.256. However, if we look at our `intercept_`, we see that it is also -0.256. Effectively, we have two redundant intercepts which, when added, are equal to 0.51."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.25602854])"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unregularized_model_intercept_true.named_steps[\"model\"].intercept_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In other words:\n",
    "\n",
    "`unregularized_model_intercept_false` was equal to $\\sigma(0 - 0.51 \\times 1 - 0.009 \\times \\text{fare} - 0.478 \\times \\text{age} - 1.06 \\times \\text{pclass} + 1.196 \\times \\text{sex}$\n",
    "\n",
    "and `unreularized_model_intercept_true` was equal to $\\sigma(0.256 - 0.256 \\times 1 - 0.009 \\times \\text{fare} - 0.478 \\times \\text{age} - 1.06 \\times \\text{pclass} + 1.196 \\times \\text{sex}$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is no difference between these models. However, things get more complicated when we have a regularized model. To understand why, let's see what happens when we remove `penalty = 'none'`, and let's pick a very small `C`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "tiny_c_model_intercept_true = Pipeline([('scale', StandardScaler()),\n",
    "                                        ('poly', PolynomialFeatures(degree=1)),\n",
    "                                        ('model',\n",
    "                                         LogisticRegression(fit_intercept=True,\n",
    "                                                            C=1e-7,\n",
    "                                                            solver='lbfgs'))])\n",
    "tiny_c_model_intercept_true.fit(p3_train[[\"fare\", \"age\", \"pclass\", \"sex\"]],\n",
    "                                p3_train[\"survived\"])\n",
    ";"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As you'd expect, the thetas generated by our model are very small, because `C` is very small."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.49126446e-13,  7.51210438e-06, -2.65108958e-06,\n",
       "        -1.03239147e-05,  1.56655731e-05]])"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tiny_c_model_intercept_true.named_steps[\"model\"].coef_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "However, the `intercept_` is NOT regularized. As you can see below, the `intercept_` is -0.377, which is enormous compared to our other coefficients."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.37776248])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tiny_c_model_intercept_true.named_steps[\"model\"].intercept_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:1510: UserWarning:\n",
      "\n",
      "Setting penalty='none' will ignore the C and l1_ratio parameters\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tiny_c_model_intercept_true_reg = Pipeline([\n",
    "    ('scale', StandardScaler()), ('poly', PolynomialFeatures(degree=1)),\n",
    "    ('model',\n",
    "     LogisticRegression(fit_intercept=True,\n",
    "                        penalty='none',\n",
    "                        C=1e-7,\n",
    "                        solver='lbfgs'))\n",
    "])\n",
    "tiny_c_model_intercept_true_reg.fit(p3_train[[\"fare\", \"age\", \"pclass\", \"sex\"]],\n",
    "                                    p3_train[\"survived\"])\n",
    ";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.25602854, -0.00927195, -0.47776545, -1.06434099,  1.19627759]])"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tiny_c_model_intercept_true_reg.named_steps[\"model\"].coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.25602854])"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tiny_c_model_intercept_true_reg.named_steps[\"model\"].intercept_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In effect, we have two intercepts, one of which is subject to regularization (and is almost equal to 0), and one of which is not."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Common convention in machine learning is to include an unregularized intercept. This gives your model additional freedom to select an intercept which minimizes the cross entropy loss.\n",
    "\n",
    "**Bottom line:** I recommend always setting `fit_intercept = True` when using a regularized model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can compare the performance of a model with regularized and unregularized intercept using the `log_loss`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "tiny_c_model_intercept_false = Pipeline([\n",
    "    ('scale', StandardScaler()), ('poly', PolynomialFeatures(degree=1)),\n",
    "    ('model', LogisticRegression(fit_intercept=False, C=1e-7, solver='lbfgs'))\n",
    "])\n",
    "tiny_c_model_intercept_false.fit(p3_train[[\"fare\", \"age\", \"pclass\", \"sex\"]],\n",
    "                                 p3_train[\"survived\"])\n",
    ";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.])"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tiny_c_model_intercept_false.named_steps[\"model\"].intercept_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-5.59991600e-06,  7.51223534e-06, -2.65118506e-06,\n",
       "        -1.03236932e-05,  1.56656209e-05]])"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tiny_c_model_intercept_false.named_steps[\"model\"].coef_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We see that cross entropy loss is slightly better when we include an unregularized intercept. We will explore this more fully on hw6."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6756154189994156"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_loss(\n",
    "    p3_train[\"survived\"],\n",
    "    tiny_c_model_intercept_true.predict_proba(\n",
    "        p3_train[[\"fare\", \"age\", \"pclass\", \"sex\"]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.693139733620562"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_loss(\n",
    "    p3_train[\"survived\"],\n",
    "    tiny_c_model_intercept_false.predict_proba(\n",
    "        p3_train[[\"fare\", \"age\", \"pclass\", \"sex\"]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
